Development merge

config/config.devnet-old.yaml

@@ -82,7 +82,6 @@ urls:
   providers: 'https://devnet-old-delegation-api.multiversx.com/providers'
   delegation: 'https://devnet-old-delegation-api.multiversx.com'
   media: 'https://devnet-old-media.elrond.com'
-  nftThumbnails: 'https://devnet-old-media.elrond.com/nfts/thumbnail'
   tmp: '/tmp'
   ipfs: 'https://ipfs.io/ipfs'
   socket: 'devnet-socket-api.multiversx.com'

config/config.devnet.yaml

@@ -52,10 +52,6 @@ features:
   assetsFetch:
     enabled: true
     assetesUrl: 'https://tools.multiversx.com/assets-cdn'
-  mediaRedirect:
-    enabled: false
-    storageUrls:
-      - 'https://s3.amazonaws.com/devnet-media.elrond.com'
   auth:
     enabled: false
     maxExpirySeconds: 86400
@@ -83,6 +79,11 @@ features:
   transactionBatch:
     enabled: true
     maxLookBehind: 100
+  elasticCircuitBreaker:
+    enabled: false
+    durationThresholdMs: 5000
+    failureCountThreshold: 5
+    resetTimeoutMs: 30000
   statusChecker:
     enabled: false
     thresholds:
@@ -133,7 +134,6 @@ urls:
   providers: 'https://devnet-delegation-api.multiversx.com/providers'
   delegation: 'https://devnet-delegation-api.multiversx.com'
   media: 'https://devnet-media.elrond.com'
-  nftThumbnails: 'https://devnet-media.elrond.com/nfts/thumbnail'
   tmp: '/tmp'
   ipfs: 'https://ipfs.io/ipfs'
   socket: 'devnet-socket-api.multiversx.com'

config/config.e2e-mocked.mainnet.yaml

@@ -34,7 +34,6 @@ urls:
   providers: 'https://internal-delegation-api.multiversx.com/providers'
   delegation: 'https://delegation-api.multiversx.com'
   media: 'https://media.elrond.com'
-  nftThumbnails: 'https://media.elrond.com/nfts/thumbnail'
   maiarId: 'https://id-api.multiversx.com'
 database:
   enabled: false

config/config.e2e.mainnet.yaml

@@ -80,6 +80,11 @@ features:
   deepHistory:
     enabled: false
     url: ''
+  elasticCircuitBreaker:
+    enabled: false
+    durationThresholdMs: 5000
+    failureCountThreshold: 5
+    resetTimeoutMs: 30000
   statusChecker:
     enabled: false
     thresholds:
@@ -133,7 +138,6 @@ urls:
   providers: 'https://delegation-api.multiversx.com/providers'
   delegation: 'https://delegation-api.multiversx.com'
   media: 'https://media.elrond.com'
-  nftThumbnails: 'https://media.elrond.com/nfts/thumbnail'
   tmp: '/tmp'
   ipfs: 'https://ipfs.io/ipfs'
   socket: 'socket-api-fra.multiversx.com'

config/config.mainnet.yaml

@@ -80,6 +80,11 @@ features:
   deepHistory:
     enabled: false
     url: ''
+  elasticCircuitBreaker:
+    enabled: false
+    durationThresholdMs: 5000
+    failureCountThreshold: 5
+    resetTimeoutMs: 30000
   statusChecker:
     enabled: false
     thresholds:
@@ -112,10 +117,6 @@ features:
   assetsFetch:
     enabled: true
     assetesUrl: 'https://tools.multiversx.com/assets-cdn'
-  mediaRedirect:
-    enabled: false
-    storageUrls:
-      - 'https://s3.amazonaws.com/media.elrond.com'
 image:
   width: 600
   height: 600
@@ -137,7 +138,6 @@ urls:
   providers: 'https://delegation-api.multiversx.com/providers'
   delegation: 'https://delegation-api.multiversx.com'
   media: 'https://media.elrond.com'
-  nftThumbnails: 'https://media.elrond.com/nfts/thumbnail'
   tmp: '/tmp'
   ipfs: 'https://ipfs.io/ipfs'
   socket: 'socket-api-fra.multiversx.com'

config/config.testnet.yaml

@@ -79,6 +79,11 @@ features:
   deepHistory:
     enabled: false
     url: ''
+  elasticCircuitBreaker:
+    enabled: false
+    durationThresholdMs: 5000
+    failureCountThreshold: 5
+    resetTimeoutMs: 30000
   statusChecker:
     enabled: false
     thresholds:
@@ -111,10 +116,6 @@ features:
   assetsFetch:
     enabled: true
     assetesUrl: 'https://tools.multiversx.com/assets-cdn'
-  mediaRedirect:
-    enabled: false
-    storageUrls:
-      - 'https://s3.amazonaws.com/testnet-media.elrond.com'
 image:
   width: 600
   height: 600
@@ -136,7 +137,6 @@ urls:
   providers: 'https://testnet-delegation-api.multiversx.com/providers'
   delegation: 'https://testnet-delegation-api.multiversx.com'
   media: 'https://testnet-media.elrond.com'
-  nftThumbnails: 'https://testnet-media.elrond.com/nfts/thumbnail'
   tmp: '/tmp'
   ipfs: 'https://ipfs.io/ipfs'
   socket: 'testnet-socket-api.multiversx.com'

entrypoint.py

@@ -96,33 +96,26 @@ def modify_yaml_variable(data, variable_name, new_value):
         return
 
 # Modify the value in the JSON structure based on the variable name
-def modify_json_variable(data, variable_name, new_value):
-    keys = variable_name[5:].split('_')  # Remove 'DAPP_' prefix
+def modify_yaml_variable(data, variable_name, new_value):
+    keys = variable_name[4:].split('_')  # Remove 'CFG_' prefix
     sub_data = data
-    
-    # Traverse the JSON structure using the keys to reach the variable and modify its value
+
+    # Traverse and create missing keys
     for key in keys[:-1]:
-        if key in sub_data:
-            sub_data = sub_data[key]
-        else:
-            print(f"Key '{key}' not found in the JSON structure.")
-            return
-
-    # Check if the value is a JSON array (list) and parse it
+        if key not in sub_data or not isinstance(sub_data[key], dict):
+            sub_data[key] = {}  # Create intermediate dict if not present
+        sub_data = sub_data[key]
+
     final_key = keys[-1]
-    if final_key in sub_data:
-        # If the new value is a string representing a JSON array, parse it
-        if isinstance(new_value, str) and new_value.startswith('[') and new_value.endswith(']'):
-            try:
-                # Parse the string as a JSON array
-                sub_data[final_key] = json.loads(new_value)
-            except json.JSONDecodeError:
-                print(f"Error decoding JSON array in value: {new_value}")
-        else:
-            sub_data[final_key] = new_value
+    # Handle array separately
+    if isinstance(new_value, str) and new_value.startswith('arr:'):
+        try:
+            sub_data[final_key] = json.loads(new_value[4:])
+        except json.JSONDecodeError:
+            print(f"Error decoding JSON array in value: {new_value}")
+            return
     else:
-        print(f"Key '{final_key}' not found at the end of the path.")
-        return
+        sub_data[final_key] = new_value
 
 # Main function
 def main():

src/common/api-config/api.config.service.ts

@@ -7,7 +7,8 @@ import { LogTopic } from '@multiversx/sdk-transaction-processor/lib/types/log-to
 
 @Injectable()
 export class ApiConfigService {
-  constructor(private readonly configService: ConfigService) { }
+  constructor(private readonly configService: ConfigService) {
+  }
 
   getConfig<T>(configKey: string): T | undefined {
     return this.configService.get<T>(configKey);
@@ -389,6 +390,23 @@ export class ApiConfigService {
     return isApiActive;
   }
 
+  isElasticCircuitBreakerEnabled(): boolean {
+    const isEnabled = this.configService.get<boolean>('features.elasticCircuitBreaker.enabled');
+    return isEnabled !== undefined ? isEnabled : false;
+  }
+
+  getElasticCircuitBreakerConfig(): {
+    durationThresholdMs: number,
+    failureCountThreshold: number,
+    resetTimeoutMs: number
+  } {
+    return {
+      durationThresholdMs: this.configService.get<number>('features.elasticCircuitBreaker.durationThresholdMs') ?? 5000,
+      failureCountThreshold: this.configService.get<number>('features.elasticCircuitBreaker.failureCountThreshold') ?? 5000,
+      resetTimeoutMs: this.configService.get<number>('features.elasticCircuitBreaker.resetTimeoutMs') ?? 5000,
+    };
+  }
+
   getIsWebsocketApiActive(): boolean {
     return this.configService.get<boolean>('api.websocket') ?? true;
   }
@@ -585,15 +603,6 @@ export class ApiConfigService {
     return mediaUrl;
   }
 
-  getNftThumbnailsUrl(): string {
-    const nftThumbnailsUrl = this.configService.get<string>('urls.nftThumbnails');
-    if (!nftThumbnailsUrl) {
-      throw new Error('No nft thumbnails url present');
-    }
-
-    return nftThumbnailsUrl;
-  }
-
   getSecurityAdmins(): string[] {
     const admins = this.configService.get<string[]>('features.auth.admins') ?? this.configService.get<string[]>('security.admins');
     if (admins === undefined) {
@@ -920,12 +929,4 @@ export class ApiConfigService {
   getCacheDuration(): number {
     return this.configService.get<number>('caching.cacheDuration') ?? 3;
   }
-
-  isMediaRedirectFeatureEnabled(): boolean {
-    return this.configService.get<boolean>('features.mediaRedirect.enabled') ?? false;
-  }
-
-  getMediaRedirectFileStorageUrls(): string[] {
-    return this.configService.get<string[]>('features.mediaRedirect.storageUrls') ?? [];
-  }
 }

src/common/indexer/elastic/circuit-breaker/circuit.breaker.proxy.module.ts

@@ -0,0 +1,15 @@
+import { Global, Module } from "@nestjs/common";
+import { ApiConfigModule } from "src/common/api-config/api.config.module";
+import { DynamicModuleUtils } from "src/utils/dynamic.module.utils";
+import { EsCircuitBreakerProxy } from "./circuit.breaker.proxy.service";
+
+@Global()
+@Module({
+  imports: [
+    ApiConfigModule,
+    DynamicModuleUtils.getElasticModule(),
+  ],
+  providers: [EsCircuitBreakerProxy],
+  exports: [EsCircuitBreakerProxy],
+})
+export class EsCircuitBreakerProxyModule { }

src/common/indexer/elastic/circuit-breaker/circuit.breaker.proxy.service.ts

@@ -0,0 +1,109 @@
+import { OriginLogger } from "@multiversx/sdk-nestjs-common";
+import { ElasticQuery, ElasticService } from "@multiversx/sdk-nestjs-elastic";
+import { Injectable, ServiceUnavailableException } from "@nestjs/common";
+import { ApiConfigService } from "../../../api-config/api.config.service";
+
+@Injectable()
+export class EsCircuitBreakerProxy {
+  private failureCount = 0;
+  private lastFailureTime = 0;
+  private isCircuitOpen = false;
+  private readonly logger = new OriginLogger(EsCircuitBreakerProxy.name);
+  private readonly enabled: boolean;
+  private readonly config: { durationThresholdMs: number, failureCountThreshold: number, resetTimeoutMs: number };
+
+  constructor(
+    readonly apiConfigService: ApiConfigService,
+    private readonly elasticService: ElasticService,
+  ) {
+    this.enabled = apiConfigService.isElasticCircuitBreakerEnabled();
+    this.config = apiConfigService.getElasticCircuitBreakerConfig();
+    this.logger.log(`ES Circuit Breaker. Enabled: ${this.enabled}. Duration threshold: ${this.config.durationThresholdMs}ms. 
+    FailureCountThreshold: ${this.config.failureCountThreshold}ms. FailureCountThreshold: ${this.config.failureCountThreshold}`);
+  }
+
+  private async withCircuitBreaker<T>(operation: () => Promise<T>): Promise<T> {
+    if (!this.enabled) {
+      return operation();
+    }
+
+    if (this.isCircuitOpen) {
+      const now = Date.now();
+      if (now - this.lastFailureTime >= this.config.resetTimeoutMs) {
+        this.logger.log('Circuit is half-open, attempting reset');
+        this.isCircuitOpen = false;
+        this.failureCount = 0;
+      } else {
+        throw new ServiceUnavailableException();
+      }
+    }
+
+    try {
+      const timeoutPromise = new Promise<never>((_, reject) => {
+        setTimeout(() => reject(new Error('Operation timed out')), this.config.durationThresholdMs);
+      });
+
+      const result = await Promise.race([operation(), timeoutPromise]);
+      this.failureCount = 0;
+      return result;
+    } catch (error) {
+      this.failureCount++;
+      this.lastFailureTime = Date.now();
+
+      if (this.failureCount >= this.config.failureCountThreshold) {
+        if (!this.isCircuitOpen) {
+          this.logger.log('Circuit breaker opened due to multiple failures');
+        }
+
+        this.isCircuitOpen = true;
+      }
+
+      throw new ServiceUnavailableException();
+    }
+  }
+
+  // eslint-disable-next-line require-await
+  async getCount(index: string, query: ElasticQuery): Promise<number> {
+    return this.withCircuitBreaker(() => this.elasticService.getCount(index, query));
+  }
+
+  // eslint-disable-next-line require-await
+  async getList(index: string, id: string, query: ElasticQuery): Promise<any[]> {
+    return this.withCircuitBreaker(() => this.elasticService.getList(index, id, query));
+  }
+
+  // eslint-disable-next-line require-await
+  async getItem(index: string, id: string, value: string): Promise<any> {
+    return this.withCircuitBreaker(() => this.elasticService.getItem(index, id, value));
+  }
+
+  // eslint-disable-next-line require-await
+  async getCustomValue(index: string, id: string, key: string): Promise<any> {
+    return this.withCircuitBreaker(() => this.elasticService.getCustomValue(index, id, key));
+  }
+
+  // eslint-disable-next-line require-await
+  async setCustomValue(index: string, id: string, key: string, value: any): Promise<void> {
+    return this.withCircuitBreaker(() => this.elasticService.setCustomValue(index, id, key, value));
+  }
+
+  // eslint-disable-next-line require-await
+  async setCustomValues(index: string, id: string, values: Record<string, any>): Promise<void> {
+    return this.withCircuitBreaker(() => this.elasticService.setCustomValues(index, id, values));
+  }
+
+  // eslint-disable-next-line require-await
+  async getScrollableList(index: string, id: string, query: ElasticQuery, action: (items: any[]) => Promise<void>): Promise<void> {
+    return this.withCircuitBreaker(() => this.elasticService.getScrollableList(index, id, query, action));
+  }
+
+  // eslint-disable-next-line require-await
+  async get(url: string): Promise<any> {
+    return this.withCircuitBreaker(() => this.elasticService.get(url));
+  }
+
+  // eslint-disable-next-line require-await
+  async post(url: string, data: any): Promise<any> {
+    return this.withCircuitBreaker(() => this.elasticService.post(url, data));
+  }
+}

src/common/indexer/elastic/elastic.indexer.module.ts

@@ -1,16 +1,16 @@
 import { forwardRef, Global, Module } from "@nestjs/common";
 import { ApiConfigModule } from "src/common/api-config/api.config.module";
 import { BlsModule } from "src/endpoints/bls/bls.module";
-import { DynamicModuleUtils } from "src/utils/dynamic.module.utils";
 import { ElasticIndexerHelper } from "./elastic.indexer.helper";
 import { ElasticIndexerService } from "./elastic.indexer.service";
+import { EsCircuitBreakerProxyModule } from "./circuit-breaker/circuit.breaker.proxy.module";
 
 @Global()
 @Module({
   imports: [
     ApiConfigModule,
     forwardRef(() => BlsModule),
-    DynamicModuleUtils.getElasticModule(),
+    EsCircuitBreakerProxyModule,
   ],
   providers: [ElasticIndexerService, ElasticIndexerHelper],
   exports: [ElasticIndexerService, ElasticIndexerHelper],