diff --git a/config/config.devnet-old.yaml b/config/config.devnet-old.yaml index b066116c1..b16959441 100644 --- a/config/config.devnet-old.yaml +++ b/config/config.devnet-old.yaml @@ -82,7 +82,6 @@ urls: providers: 'https://devnet-old-delegation-api.multiversx.com/providers' delegation: 'https://devnet-old-delegation-api.multiversx.com' media: 'https://devnet-old-media.elrond.com' - nftThumbnails: 'https://devnet-old-media.elrond.com/nfts/thumbnail' tmp: '/tmp' ipfs: 'https://ipfs.io/ipfs' socket: 'devnet-socket-api.multiversx.com' diff --git a/config/config.devnet.yaml b/config/config.devnet.yaml index c00ca7f39..33cbd8382 100644 --- a/config/config.devnet.yaml +++ b/config/config.devnet.yaml @@ -52,10 +52,6 @@ features: assetsFetch: enabled: true assetesUrl: 'https://tools.multiversx.com/assets-cdn' - mediaRedirect: - enabled: false - storageUrls: - - 'https://s3.amazonaws.com/devnet-media.elrond.com' auth: enabled: false maxExpirySeconds: 86400 @@ -83,6 +79,11 @@ features: transactionBatch: enabled: true maxLookBehind: 100 + elasticCircuitBreaker: + enabled: false + durationThresholdMs: 5000 + failureCountThreshold: 5 + resetTimeoutMs: 30000 statusChecker: enabled: false thresholds: @@ -133,7 +134,6 @@ urls: providers: 'https://devnet-delegation-api.multiversx.com/providers' delegation: 'https://devnet-delegation-api.multiversx.com' media: 'https://devnet-media.elrond.com' - nftThumbnails: 'https://devnet-media.elrond.com/nfts/thumbnail' tmp: '/tmp' ipfs: 'https://ipfs.io/ipfs' socket: 'devnet-socket-api.multiversx.com' diff --git a/config/config.e2e-mocked.mainnet.yaml b/config/config.e2e-mocked.mainnet.yaml index f2ffbd055..ef1cd3eed 100644 --- a/config/config.e2e-mocked.mainnet.yaml +++ b/config/config.e2e-mocked.mainnet.yaml @@ -34,7 +34,6 @@ urls: providers: 'https://internal-delegation-api.multiversx.com/providers' delegation: 'https://delegation-api.multiversx.com' media: 'https://media.elrond.com' - nftThumbnails: 'https://media.elrond.com/nfts/thumbnail' maiarId: 'https://id-api.multiversx.com' database: enabled: false diff --git a/config/config.e2e.mainnet.yaml b/config/config.e2e.mainnet.yaml index 57b050a6e..bbecedd63 100644 --- a/config/config.e2e.mainnet.yaml +++ b/config/config.e2e.mainnet.yaml @@ -80,6 +80,11 @@ features: deepHistory: enabled: false url: '' + elasticCircuitBreaker: + enabled: false + durationThresholdMs: 5000 + failureCountThreshold: 5 + resetTimeoutMs: 30000 statusChecker: enabled: false thresholds: @@ -133,7 +138,6 @@ urls: providers: 'https://delegation-api.multiversx.com/providers' delegation: 'https://delegation-api.multiversx.com' media: 'https://media.elrond.com' - nftThumbnails: 'https://media.elrond.com/nfts/thumbnail' tmp: '/tmp' ipfs: 'https://ipfs.io/ipfs' socket: 'socket-api-fra.multiversx.com' diff --git a/config/config.mainnet.yaml b/config/config.mainnet.yaml index d0b9035d0..a667fa71c 100644 --- a/config/config.mainnet.yaml +++ b/config/config.mainnet.yaml @@ -80,6 +80,11 @@ features: deepHistory: enabled: false url: '' + elasticCircuitBreaker: + enabled: false + durationThresholdMs: 5000 + failureCountThreshold: 5 + resetTimeoutMs: 30000 statusChecker: enabled: false thresholds: @@ -112,10 +117,6 @@ features: assetsFetch: enabled: true assetesUrl: 'https://tools.multiversx.com/assets-cdn' - mediaRedirect: - enabled: false - storageUrls: - - 'https://s3.amazonaws.com/media.elrond.com' image: width: 600 height: 600 @@ -137,7 +138,6 @@ urls: providers: 'https://delegation-api.multiversx.com/providers' delegation: 'https://delegation-api.multiversx.com' media: 'https://media.elrond.com' - nftThumbnails: 'https://media.elrond.com/nfts/thumbnail' tmp: '/tmp' ipfs: 'https://ipfs.io/ipfs' socket: 'socket-api-fra.multiversx.com' diff --git a/config/config.testnet.yaml b/config/config.testnet.yaml index eaf43d498..a4104d9ec 100644 --- a/config/config.testnet.yaml +++ b/config/config.testnet.yaml @@ -79,6 +79,11 @@ features: deepHistory: enabled: false url: '' + elasticCircuitBreaker: + enabled: false + durationThresholdMs: 5000 + failureCountThreshold: 5 + resetTimeoutMs: 30000 statusChecker: enabled: false thresholds: @@ -111,10 +116,6 @@ features: assetsFetch: enabled: true assetesUrl: 'https://tools.multiversx.com/assets-cdn' - mediaRedirect: - enabled: false - storageUrls: - - 'https://s3.amazonaws.com/testnet-media.elrond.com' image: width: 600 height: 600 @@ -136,7 +137,6 @@ urls: providers: 'https://testnet-delegation-api.multiversx.com/providers' delegation: 'https://testnet-delegation-api.multiversx.com' media: 'https://testnet-media.elrond.com' - nftThumbnails: 'https://testnet-media.elrond.com/nfts/thumbnail' tmp: '/tmp' ipfs: 'https://ipfs.io/ipfs' socket: 'testnet-socket-api.multiversx.com' diff --git a/entrypoint.py b/entrypoint.py index 1a7ad344b..87a189890 100644 --- a/entrypoint.py +++ b/entrypoint.py @@ -96,33 +96,26 @@ def modify_yaml_variable(data, variable_name, new_value): return # Modify the value in the JSON structure based on the variable name -def modify_json_variable(data, variable_name, new_value): - keys = variable_name[5:].split('_') # Remove 'DAPP_' prefix +def modify_yaml_variable(data, variable_name, new_value): + keys = variable_name[4:].split('_') # Remove 'CFG_' prefix sub_data = data - - # Traverse the JSON structure using the keys to reach the variable and modify its value + + # Traverse and create missing keys for key in keys[:-1]: - if key in sub_data: - sub_data = sub_data[key] - else: - print(f"Key '{key}' not found in the JSON structure.") - return - - # Check if the value is a JSON array (list) and parse it + if key not in sub_data or not isinstance(sub_data[key], dict): + sub_data[key] = {} # Create intermediate dict if not present + sub_data = sub_data[key] + final_key = keys[-1] - if final_key in sub_data: - # If the new value is a string representing a JSON array, parse it - if isinstance(new_value, str) and new_value.startswith('[') and new_value.endswith(']'): - try: - # Parse the string as a JSON array - sub_data[final_key] = json.loads(new_value) - except json.JSONDecodeError: - print(f"Error decoding JSON array in value: {new_value}") - else: - sub_data[final_key] = new_value + # Handle array separately + if isinstance(new_value, str) and new_value.startswith('arr:'): + try: + sub_data[final_key] = json.loads(new_value[4:]) + except json.JSONDecodeError: + print(f"Error decoding JSON array in value: {new_value}") + return else: - print(f"Key '{final_key}' not found at the end of the path.") - return + sub_data[final_key] = new_value # Main function def main(): diff --git a/src/common/api-config/api.config.service.ts b/src/common/api-config/api.config.service.ts index 07ba50c2a..150f740ac 100644 --- a/src/common/api-config/api.config.service.ts +++ b/src/common/api-config/api.config.service.ts @@ -7,7 +7,8 @@ import { LogTopic } from '@multiversx/sdk-transaction-processor/lib/types/log-to @Injectable() export class ApiConfigService { - constructor(private readonly configService: ConfigService) { } + constructor(private readonly configService: ConfigService) { + } getConfig(configKey: string): T | undefined { return this.configService.get(configKey); @@ -389,6 +390,23 @@ export class ApiConfigService { return isApiActive; } + isElasticCircuitBreakerEnabled(): boolean { + const isEnabled = this.configService.get('features.elasticCircuitBreaker.enabled'); + return isEnabled !== undefined ? isEnabled : false; + } + + getElasticCircuitBreakerConfig(): { + durationThresholdMs: number, + failureCountThreshold: number, + resetTimeoutMs: number + } { + return { + durationThresholdMs: this.configService.get('features.elasticCircuitBreaker.durationThresholdMs') ?? 5000, + failureCountThreshold: this.configService.get('features.elasticCircuitBreaker.failureCountThreshold') ?? 5000, + resetTimeoutMs: this.configService.get('features.elasticCircuitBreaker.resetTimeoutMs') ?? 5000, + }; + } + getIsWebsocketApiActive(): boolean { return this.configService.get('api.websocket') ?? true; } @@ -585,15 +603,6 @@ export class ApiConfigService { return mediaUrl; } - getNftThumbnailsUrl(): string { - const nftThumbnailsUrl = this.configService.get('urls.nftThumbnails'); - if (!nftThumbnailsUrl) { - throw new Error('No nft thumbnails url present'); - } - - return nftThumbnailsUrl; - } - getSecurityAdmins(): string[] { const admins = this.configService.get('features.auth.admins') ?? this.configService.get('security.admins'); if (admins === undefined) { @@ -920,12 +929,4 @@ export class ApiConfigService { getCacheDuration(): number { return this.configService.get('caching.cacheDuration') ?? 3; } - - isMediaRedirectFeatureEnabled(): boolean { - return this.configService.get('features.mediaRedirect.enabled') ?? false; - } - - getMediaRedirectFileStorageUrls(): string[] { - return this.configService.get('features.mediaRedirect.storageUrls') ?? []; - } } diff --git a/src/common/indexer/elastic/circuit-breaker/circuit.breaker.proxy.module.ts b/src/common/indexer/elastic/circuit-breaker/circuit.breaker.proxy.module.ts new file mode 100644 index 000000000..86b124a65 --- /dev/null +++ b/src/common/indexer/elastic/circuit-breaker/circuit.breaker.proxy.module.ts @@ -0,0 +1,15 @@ +import { Global, Module } from "@nestjs/common"; +import { ApiConfigModule } from "src/common/api-config/api.config.module"; +import { DynamicModuleUtils } from "src/utils/dynamic.module.utils"; +import { EsCircuitBreakerProxy } from "./circuit.breaker.proxy.service"; + +@Global() +@Module({ + imports: [ + ApiConfigModule, + DynamicModuleUtils.getElasticModule(), + ], + providers: [EsCircuitBreakerProxy], + exports: [EsCircuitBreakerProxy], +}) +export class EsCircuitBreakerProxyModule { } diff --git a/src/common/indexer/elastic/circuit-breaker/circuit.breaker.proxy.service.ts b/src/common/indexer/elastic/circuit-breaker/circuit.breaker.proxy.service.ts new file mode 100644 index 000000000..7cff25aea --- /dev/null +++ b/src/common/indexer/elastic/circuit-breaker/circuit.breaker.proxy.service.ts @@ -0,0 +1,109 @@ +import { OriginLogger } from "@multiversx/sdk-nestjs-common"; +import { ElasticQuery, ElasticService } from "@multiversx/sdk-nestjs-elastic"; +import { Injectable, ServiceUnavailableException } from "@nestjs/common"; +import { ApiConfigService } from "../../../api-config/api.config.service"; + +@Injectable() +export class EsCircuitBreakerProxy { + private failureCount = 0; + private lastFailureTime = 0; + private isCircuitOpen = false; + private readonly logger = new OriginLogger(EsCircuitBreakerProxy.name); + private readonly enabled: boolean; + private readonly config: { durationThresholdMs: number, failureCountThreshold: number, resetTimeoutMs: number }; + + constructor( + readonly apiConfigService: ApiConfigService, + private readonly elasticService: ElasticService, + ) { + this.enabled = apiConfigService.isElasticCircuitBreakerEnabled(); + this.config = apiConfigService.getElasticCircuitBreakerConfig(); + this.logger.log(`ES Circuit Breaker. Enabled: ${this.enabled}. Duration threshold: ${this.config.durationThresholdMs}ms. + FailureCountThreshold: ${this.config.failureCountThreshold}ms. FailureCountThreshold: ${this.config.failureCountThreshold}`); + } + + private async withCircuitBreaker(operation: () => Promise): Promise { + if (!this.enabled) { + return operation(); + } + + if (this.isCircuitOpen) { + const now = Date.now(); + if (now - this.lastFailureTime >= this.config.resetTimeoutMs) { + this.logger.log('Circuit is half-open, attempting reset'); + this.isCircuitOpen = false; + this.failureCount = 0; + } else { + throw new ServiceUnavailableException(); + } + } + + try { + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => reject(new Error('Operation timed out')), this.config.durationThresholdMs); + }); + + const result = await Promise.race([operation(), timeoutPromise]); + this.failureCount = 0; + return result; + } catch (error) { + this.failureCount++; + this.lastFailureTime = Date.now(); + + if (this.failureCount >= this.config.failureCountThreshold) { + if (!this.isCircuitOpen) { + this.logger.log('Circuit breaker opened due to multiple failures'); + } + + this.isCircuitOpen = true; + } + + throw new ServiceUnavailableException(); + } + } + + // eslint-disable-next-line require-await + async getCount(index: string, query: ElasticQuery): Promise { + return this.withCircuitBreaker(() => this.elasticService.getCount(index, query)); + } + + // eslint-disable-next-line require-await + async getList(index: string, id: string, query: ElasticQuery): Promise { + return this.withCircuitBreaker(() => this.elasticService.getList(index, id, query)); + } + + // eslint-disable-next-line require-await + async getItem(index: string, id: string, value: string): Promise { + return this.withCircuitBreaker(() => this.elasticService.getItem(index, id, value)); + } + + // eslint-disable-next-line require-await + async getCustomValue(index: string, id: string, key: string): Promise { + return this.withCircuitBreaker(() => this.elasticService.getCustomValue(index, id, key)); + } + + // eslint-disable-next-line require-await + async setCustomValue(index: string, id: string, key: string, value: any): Promise { + return this.withCircuitBreaker(() => this.elasticService.setCustomValue(index, id, key, value)); + } + + // eslint-disable-next-line require-await + async setCustomValues(index: string, id: string, values: Record): Promise { + return this.withCircuitBreaker(() => this.elasticService.setCustomValues(index, id, values)); + } + + // eslint-disable-next-line require-await + async getScrollableList(index: string, id: string, query: ElasticQuery, action: (items: any[]) => Promise): Promise { + return this.withCircuitBreaker(() => this.elasticService.getScrollableList(index, id, query, action)); + } + + // eslint-disable-next-line require-await + async get(url: string): Promise { + return this.withCircuitBreaker(() => this.elasticService.get(url)); + } + + // eslint-disable-next-line require-await + async post(url: string, data: any): Promise { + return this.withCircuitBreaker(() => this.elasticService.post(url, data)); + } +} diff --git a/src/common/indexer/elastic/elastic.indexer.module.ts b/src/common/indexer/elastic/elastic.indexer.module.ts index 254fcae40..e95d46720 100644 --- a/src/common/indexer/elastic/elastic.indexer.module.ts +++ b/src/common/indexer/elastic/elastic.indexer.module.ts @@ -1,16 +1,16 @@ import { forwardRef, Global, Module } from "@nestjs/common"; import { ApiConfigModule } from "src/common/api-config/api.config.module"; import { BlsModule } from "src/endpoints/bls/bls.module"; -import { DynamicModuleUtils } from "src/utils/dynamic.module.utils"; import { ElasticIndexerHelper } from "./elastic.indexer.helper"; import { ElasticIndexerService } from "./elastic.indexer.service"; +import { EsCircuitBreakerProxyModule } from "./circuit-breaker/circuit.breaker.proxy.module"; @Global() @Module({ imports: [ ApiConfigModule, forwardRef(() => BlsModule), - DynamicModuleUtils.getElasticModule(), + EsCircuitBreakerProxyModule, ], providers: [ElasticIndexerService, ElasticIndexerHelper], exports: [ElasticIndexerService, ElasticIndexerHelper], diff --git a/src/common/indexer/elastic/elastic.indexer.service.ts b/src/common/indexer/elastic/elastic.indexer.service.ts index 9188341cf..ada83218e 100644 --- a/src/common/indexer/elastic/elastic.indexer.service.ts +++ b/src/common/indexer/elastic/elastic.indexer.service.ts @@ -1,6 +1,6 @@ import { HttpStatus, Injectable } from "@nestjs/common"; import { BinaryUtils } from "@multiversx/sdk-nestjs-common"; -import { ElasticService, ElasticQuery, QueryOperator, QueryType, QueryConditionOptions, ElasticSortOrder, ElasticSortProperty, TermsQuery, RangeGreaterThanOrEqual, MatchQuery } from "@multiversx/sdk-nestjs-elastic"; +import { ElasticQuery, QueryOperator, QueryType, QueryConditionOptions, ElasticSortOrder, ElasticSortProperty, TermsQuery, RangeGreaterThanOrEqual, MatchQuery } from "@multiversx/sdk-nestjs-elastic"; import { IndexerInterface } from "../indexer.interface"; import { ApiConfigService } from "src/common/api-config/api.config.service"; import { CollectionFilter } from "src/endpoints/collections/entities/collection.filter"; @@ -29,6 +29,7 @@ import { ApplicationFilter } from "src/endpoints/applications/entities/applicati import { NftType } from "../entities/nft.type"; import { EventsFilter } from "src/endpoints/events/entities/events.filter"; import { Events } from "../entities/events"; +import { EsCircuitBreakerProxy } from "./circuit-breaker/circuit.breaker.proxy.service"; @Injectable() export class ElasticIndexerService implements IndexerInterface { @@ -38,7 +39,7 @@ export class ElasticIndexerService implements IndexerInterface { constructor( private readonly apiConfigService: ApiConfigService, - private readonly elasticService: ElasticService, + private readonly elasticService: EsCircuitBreakerProxy, private readonly indexerHelper: ElasticIndexerHelper, ) { } diff --git a/src/endpoints/endpoints.controllers.module.ts b/src/endpoints/endpoints.controllers.module.ts index de157bc75..318e527f0 100644 --- a/src/endpoints/endpoints.controllers.module.ts +++ b/src/endpoints/endpoints.controllers.module.ts @@ -39,7 +39,6 @@ import { PoolController } from "./pool/pool.controller"; import { TpsController } from "./tps/tps.controller"; import { ApplicationController } from "./applications/application.controller"; import { EventsController } from "./events/events.controller"; -import { MediaController } from "./media/media.controller"; @Module({}) export class EndpointsControllersModule { @@ -51,7 +50,6 @@ export class EndpointsControllersModule { TokenController, TransactionController, UsernameController, VmQueryController, WaitingListController, HealthCheckController, DappConfigController, WebsocketController, TransferController, ProcessNftsPublicController, TransactionsBatchController, ApplicationController, EventsController, - MediaController, ]; const isMarketplaceFeatureEnabled = configuration().features?.marketplace?.enabled ?? false; diff --git a/src/endpoints/endpoints.services.module.ts b/src/endpoints/endpoints.services.module.ts index 7c56d4b5a..fc8531828 100644 --- a/src/endpoints/endpoints.services.module.ts +++ b/src/endpoints/endpoints.services.module.ts @@ -36,7 +36,6 @@ import { PoolModule } from "./pool/pool.module"; import { TpsModule } from "./tps/tps.module"; import { ApplicationModule } from "./applications/application.module"; import { EventsModule } from "./events/events.module"; -import { MediaModule } from "./media/media.module"; @Module({ imports: [ @@ -78,14 +77,13 @@ import { MediaModule } from "./media/media.module"; TpsModule, ApplicationModule, EventsModule, - MediaModule, ], exports: [ AccountModule, CollectionModule, BlockModule, DelegationModule, DelegationLegacyModule, IdentitiesModule, KeysModule, MiniBlockModule, NetworkModule, NftModule, NftMediaModule, TagModule, NodeModule, ProviderModule, RoundModule, SmartContractResultModule, ShardModule, StakeModule, TokenModule, RoundModule, TransactionModule, UsernameModule, VmQueryModule, WaitingListModule, EsdtModule, BlsModule, DappConfigModule, TransferModule, PoolModule, TransactionActionModule, WebsocketModule, MexModule, - ProcessNftsModule, NftMarketplaceModule, TransactionsBatchModule, TpsModule, ApplicationModule, EventsModule, MediaModule, + ProcessNftsModule, NftMarketplaceModule, TransactionsBatchModule, TpsModule, ApplicationModule, EventsModule, ], }) export class EndpointsServicesModule { } diff --git a/src/endpoints/media/media.controller.ts b/src/endpoints/media/media.controller.ts deleted file mode 100644 index e4ece3ead..000000000 --- a/src/endpoints/media/media.controller.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { Controller, Get, InternalServerErrorException, NotFoundException, Param, Req, Res } from "@nestjs/common"; -import { ApiTags } from "@nestjs/swagger"; -import { Request, Response } from "express"; -import { MediaService } from "./media.service"; -import { ApiService } from "@multiversx/sdk-nestjs-http"; -import { OriginLogger } from "@multiversx/sdk-nestjs-common"; -import https from 'https'; - -@Controller() -@ApiTags('media') -export class MediaController { - private readonly logger = new OriginLogger(MediaController.name); - - constructor( - private readonly mediaService: MediaService, - private readonly apiService: ApiService, - ) { } - - @Get("/media/:uri(*)") - async redirectToMediaUri( - @Param('uri') uri: string, - @Req() req: Request, - @Res() res: Response - ) { - const redirectUrl = await this.mediaService.getRedirectUrl(uri); - if (!redirectUrl) { - throw new NotFoundException('Not found'); - } - - try { - const response = await this.apiService.get(redirectUrl, { - // @ts-ignore - responseType: 'stream', - timeout: 60_000, // 60 seconds timeout - httpsAgent: new https.Agent({ rejectUnauthorized: false }), - headers: { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36', - }, - }); - - res.setHeader('content-type', response.headers['content-type']); - res.setHeader('content-length', response.headers['content-length']); - res.setHeader('cache-control', 'max-age=60'); - res.setHeader('Access-Control-Allow-Origin', '*'); - - response.data.pipe(res); - - response.data.on('error', (error: any) => { - this.logger?.error(`Error streaming the resource: ${redirectUrl}`); - this.logger?.error(error); - - throw new InternalServerErrorException('Failed to fetch URL'); - }); - - req.on('close', () => { - response.data?.destroy(); - }); - - return; - } catch (error) { - this.logger.error(`Failed to fetch URL: ${redirectUrl}`); - - throw new InternalServerErrorException('Failed to fetch URL'); - } - } -} diff --git a/src/endpoints/media/media.module.ts b/src/endpoints/media/media.module.ts deleted file mode 100644 index 95360285a..000000000 --- a/src/endpoints/media/media.module.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { Module } from "@nestjs/common"; -import { MediaService } from "./media.service"; - -@Module({ - imports: [], - providers: [ - MediaService, - ], - exports: [ - MediaService, - ], -}) -export class MediaModule { } diff --git a/src/endpoints/media/media.service.ts b/src/endpoints/media/media.service.ts deleted file mode 100644 index 8f357c654..000000000 --- a/src/endpoints/media/media.service.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { OriginLogger } from "@multiversx/sdk-nestjs-common"; -import { ApiService } from "@multiversx/sdk-nestjs-http"; -import { Injectable } from "@nestjs/common"; -import { ApiConfigService } from "src/common/api-config/api.config.service"; - -@Injectable() -export class MediaService { - private readonly logger = new OriginLogger(MediaService.name); - - private readonly fallbackThumbnail = 'nfts/thumbnail/default.png'; - - constructor( - private readonly apiConfigService: ApiConfigService, - private readonly apiService: ApiService, - ) { } - - public async getRedirectUrl(uri: string): Promise { - const isFeatureEnabled = this.apiConfigService.isMediaRedirectFeatureEnabled(); - if (!isFeatureEnabled) { - // TODO: throw error - // throw new BadRequestException('Media redirect is not allowed'); - } - - // providers logos - if (uri.startsWith('providers/asset/')) { - const awsUri = uri.replace('providers/asset/', 'keybase_processed_uploads/'); - return `https://s3.amazonaws.com/${awsUri}`; - } - - // esdts logos - if (uri.startsWith('tokens/asset/')) { - const network = this.apiConfigService.getNetwork(); - const tokenUri = network === 'mainnet' - ? uri.replace('tokens/asset/', 'multiversx/mx-assets/master/tokens/') - : uri.replace('tokens/asset/', `multiversx/mx-assets/master/${network}/tokens/`); - return `https://raw.githubusercontent.com/${tokenUri}`; - } - - const fileStorageUrls = this.apiConfigService.getMediaRedirectFileStorageUrls(); - for (const fileStorageUrl of fileStorageUrls) { - try { - const { status } = await this.apiService.head(`${fileStorageUrl}/${uri}`, { - validateStatus: () => true, - }); - if (200 <= status && status <= 300) { - return `${fileStorageUrl}/${uri}`; - } - } catch { - this.logger.error(`Could not fetch ${fileStorageUrl}/${uri}`); - continue; - } - } - - // nfts assets' ipfs mirror - if (uri.startsWith('nfts/asset/')) { - const ipfsUri = uri.replace('nfts/asset/', 'ipfs/'); - return `https://ipfs.io/${ipfsUri}`; - } - - // fallback for nft thumbnails - if (uri.startsWith('nfts/thumbnail/') && uri !== this.fallbackThumbnail && fileStorageUrls.length > 0) { - return `${fileStorageUrls[0]}/${this.fallbackThumbnail}`; - } - - return undefined; - } -} diff --git a/src/endpoints/mex/mex.controller.ts b/src/endpoints/mex/mex.controller.ts index 988ef755d..b520de22a 100644 --- a/src/endpoints/mex/mex.controller.ts +++ b/src/endpoints/mex/mex.controller.ts @@ -178,13 +178,15 @@ export class MexController { } @Get('mex/tokens/prices/daily/:identifier') - @ApiOperation({ summary: 'xExchange token prices daily', description: 'Returns token prices daily' }) + @ApiOperation({ + summary: 'xExchange token prices daily', + description: 'Returns token prices daily, ordered by timestamp in ascending order. The entries represent the latest complete daily values for the given token series.', + }) @ApiOkResponse({ type: [MexTokenChart] }) @ApiNotFoundResponse({ description: 'Price not available for given token identifier' }) async getTokenPricesDayResolution( - @Param('identifier', ParseTokenPipe) identifier: string, - @Query('after') after: string): Promise { - const charts = await this.mexTokenChartsService.getTokenPricesDayResolution(identifier, after); + @Param('identifier', ParseTokenPipe) identifier: string): Promise { + const charts = await this.mexTokenChartsService.getTokenPricesDayResolution(identifier); if (!charts) { throw new NotFoundException('Price not available for given token identifier'); } diff --git a/src/endpoints/mex/mex.token.charts.service.ts b/src/endpoints/mex/mex.token.charts.service.ts index 0b3dde374..0c4090947 100644 --- a/src/endpoints/mex/mex.token.charts.service.ts +++ b/src/endpoints/mex/mex.token.charts.service.ts @@ -42,15 +42,15 @@ export class MexTokenChartsService { } } - async getTokenPricesDayResolution(tokenIdentifier: string, after: string): Promise { + async getTokenPricesDayResolution(tokenIdentifier: string): Promise { return await this.cachingService.getOrSet( - CacheInfo.TokenDailyChart(tokenIdentifier, after).key, - async () => await this.getTokenPricesDayResolutionRaw(tokenIdentifier, after), - CacheInfo.TokenDailyChart(tokenIdentifier, after).ttl, + CacheInfo.TokenDailyChart(tokenIdentifier).key, + async () => await this.getTokenPricesDayResolutionRaw(tokenIdentifier), + CacheInfo.TokenDailyChart(tokenIdentifier).ttl, ); } - async getTokenPricesDayResolutionRaw(tokenIdentifier: string, after: string): Promise { + async getTokenPricesDayResolutionRaw(tokenIdentifier: string): Promise { const isMexToken = await this.isMexToken(tokenIdentifier); if (!isMexToken) { return undefined; @@ -61,7 +61,6 @@ export class MexTokenChartsService { latestCompleteValues( series: "${tokenIdentifier}", metric: "priceUSD", - start: "${after}" ) { timestamp value diff --git a/src/endpoints/nfts/nft.service.ts b/src/endpoints/nfts/nft.service.ts index 46e651db6..91586a4a4 100644 --- a/src/endpoints/nfts/nft.service.ts +++ b/src/endpoints/nfts/nft.service.ts @@ -652,31 +652,28 @@ export class NftService { } private applyRedirectMedia(nft: Nft) { - // FIXME: This is a temporary fix to avoid breaking the API - const isMediaRedirectFeatureEnabled = this.apiConfigService.isMediaRedirectFeatureEnabled(); - if (!isMediaRedirectFeatureEnabled) { - // return; - } - - if (!nft.media || nft.media.length === 0) { + if (!nft.media || !Array.isArray(nft.media) || nft.media.length === 0) { return; } try { const network = this.apiConfigService.getNetwork(); - // const defaultMediaUrl = `https://${network === 'mainnet' ? '' : `${network}-`}media.elrond.com`; - const defaultMediaUrl = `https://${network === 'mainnet' ? '' : `${network}-`}api.multiversx.com/media`; + const defaultMediaUrl = `https://${network === 'mainnet' ? '' : `${network}-`}media.elrond.com`; + const defaultApiMediaUrl = `https://${network === 'mainnet' ? '' : `${network}-`}api.multiversx.com/media`; for (const media of nft.media) { if (media.url) { - media.url = media.url.replace(defaultMediaUrl, this.apiConfigService.getMediaUrl()); + media.url = ApiUtils.replaceUri(media.url, defaultMediaUrl, this.apiConfigService.getMediaUrl()); + media.url = ApiUtils.replaceUri(media.url, defaultApiMediaUrl, this.apiConfigService.getMediaUrl()); } if (media.thumbnailUrl) { - media.thumbnailUrl = media.thumbnailUrl.replace(defaultMediaUrl, this.apiConfigService.getMediaUrl()); + media.thumbnailUrl = ApiUtils.replaceUri(media.thumbnailUrl, defaultMediaUrl, this.apiConfigService.getMediaUrl()); + media.thumbnailUrl = ApiUtils.replaceUri(media.thumbnailUrl, defaultApiMediaUrl, this.apiConfigService.getMediaUrl()); } } - } catch { - // TODO: there are some cases where the nft.media is an empty object, we should investigate why + } catch (error) { + this.logger.error(`Error when applying redirect media for NFT with identifier '${nft.identifier}'`); + this.logger.error(error); } } } diff --git a/src/endpoints/tokens/token.service.ts b/src/endpoints/tokens/token.service.ts index b6f996dae..52d1f77da 100644 --- a/src/endpoints/tokens/token.service.ts +++ b/src/endpoints/tokens/token.service.ts @@ -733,7 +733,7 @@ export class TokenService { const tokens = await this.getAllTokens(); for (const token of tokens) { - if (token.price && token.marketCap && !token.isLowLiquidity) { + if (token.price && token.marketCap && !token.isLowLiquidity && token.assets?.priceSource?.type !== TokenAssetsPriceSourceType.customUrl) { totalMarketCap += token.marketCap; } } @@ -837,7 +837,8 @@ export class TokenService { tokens = tokens.sortedDescending( token => token.assets ? 1 : 0, - token => token.isLowLiquidity ? 0 : (token.marketCap ?? 0), + token => token.marketCap ? 1 : 0, + token => token.isLowLiquidity || token.assets?.priceSource?.type === TokenAssetsPriceSourceType.customUrl ? 0 : (token.marketCap ?? 0), token => token.transactions ?? 0, ); diff --git a/src/test/unit/controllers/media.controller.spec.ts b/src/test/unit/controllers/media.controller.spec.ts deleted file mode 100644 index 1570042d4..000000000 --- a/src/test/unit/controllers/media.controller.spec.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { INestApplication } from "@nestjs/common"; -import { Test } from "@nestjs/testing"; -import request = require('supertest'); -import { MediaController } from "src/endpoints/media/media.controller"; -import { PublicAppModule } from "src/public.app.module"; -import { MediaService } from "src/endpoints/media/media.service"; - -describe('MediaController', () => { - let app: INestApplication; - const path: string = "/media"; - - const mediaService = { - getRedirectUrl: jest.fn(), - }; - - beforeAll(async () => { - const moduleRef = await Test.createTestingModule({ - controllers: [MediaController], - imports: [PublicAppModule], - }) - .overrideProvider(MediaService) - .useValue(mediaService) - .compile(); - - app = moduleRef.createNestApplication(); - await app.init(); - }); - - it(`/GET media/:uri(*)`, async () => { - const mockUrl = 'https://s3.amazonaws.com/media.elrond.com/nfts/thumbnail/XPACHIEVE-5a0519-e302a15d'; - - mediaService.getRedirectUrl.mockResolvedValue(mockUrl); - - await request(app.getHttpServer()) - .get(`${path}/nfts/thumbnail/XPACHIEVE-5a0519-e302a15d`) - .expect(200) - .expect('content-type', 'image/jpeg') - .expect('cache-control', 'max-age=60') - .expect('Access-Control-Allow-Origin', '*'); - - expect(mediaService.getRedirectUrl).toHaveBeenCalled(); - }); - - it(`/GET media/:uri(*) - not found`, async () => { - mediaService.getRedirectUrl.mockResolvedValue(undefined); - - await request(app.getHttpServer()) - .get(`${path}/nfts/thumbnail/XPACHIEVE-5a0519-e302a15d`) - .expect(404); - }); - - afterAll(async () => { - await app.close(); - }); -}); diff --git a/src/test/unit/controllers/services.mock/account.services.mock.ts b/src/test/unit/controllers/services.mock/account.services.mock.ts index 5eb918811..df379edfd 100644 --- a/src/test/unit/controllers/services.mock/account.services.mock.ts +++ b/src/test/unit/controllers/services.mock/account.services.mock.ts @@ -76,6 +76,8 @@ export const mockApiConfigService = () => ({ getProcessTtl: jest.fn().mockReturnValue(''), getExternalMediaUrl: jest.fn().mockReturnValue(''), getMediaUrl: jest.fn().mockReturnValue(''), + isElasticCircuitBreakerEnabled: jest.fn().mockReturnValue(false), + getElasticCircuitBreakerConfig: jest.fn().mockReturnValue({}), getConfig: jest.fn(), }); diff --git a/src/test/unit/services/api.config.spec.ts b/src/test/unit/services/api.config.spec.ts index a0c5650d8..f21a50c2e 100644 --- a/src/test/unit/services/api.config.spec.ts +++ b/src/test/unit/services/api.config.spec.ts @@ -1125,25 +1125,6 @@ describe('API Config', () => { }); }); - describe("getNftThumbnailsUrl", () => { - it("should return nft thumbnails url", () => { - jest - .spyOn(ConfigService.prototype, "get") - .mockImplementation(jest.fn(() => 'https://media.elrond.com/nfts/thumbnail')); - - const results = apiConfigService.getNftThumbnailsUrl(); - expect(results).toEqual('https://media.elrond.com/nfts/thumbnail'); - }); - - it("should throw error because test simulates that nft thumbnails urls are not defined", () => { - jest - .spyOn(ConfigService.prototype, 'get') - .mockImplementation(jest.fn(() => undefined)); - - expect(() => apiConfigService.getNftThumbnailsUrl()).toThrowError('No nft thumbnails url present'); - }); - }); - describe("getSecurityAdmins", () => { it("should return nft thumbnails url", () => { jest diff --git a/src/test/unit/services/media.spec.ts b/src/test/unit/services/media.spec.ts deleted file mode 100644 index da602aeae..000000000 --- a/src/test/unit/services/media.spec.ts +++ /dev/null @@ -1,135 +0,0 @@ -import { ApiService } from "@multiversx/sdk-nestjs-http"; -import { Test } from "@nestjs/testing"; -import { ApiConfigService } from "src/common/api-config/api.config.service"; -import { MediaService } from "src/endpoints/media/media.service"; - -describe('MediaService', () => { - let mediaService: MediaService; - let apiConfigService: ApiConfigService; - let apiService: ApiService; - - beforeEach(async () => { - const moduleRef = await Test.createTestingModule({ - providers: [ - MediaService, - { - provide: ApiConfigService, - useValue: { - getNetwork: jest.fn(), - isMediaRedirectFeatureEnabled: jest.fn(), - getMediaRedirectFileStorageUrls: jest.fn(), - }, - }, - { - provide: ApiService, - useValue: { - head: jest.fn(), - }, - }, - ], - }).compile(); - - mediaService = moduleRef.get(MediaService); - apiConfigService = moduleRef.get(ApiConfigService); - apiService = moduleRef.get(ApiService); - }); - - describe('redirectToMediaUri', () => { - it.skip('should throw BadRequestException when media redirect feature is disabled', async () => { - jest.spyOn(apiConfigService, 'isMediaRedirectFeatureEnabled').mockReturnValueOnce(false); - - await expect(mediaService.getRedirectUrl('url')).rejects.toThrowError('Media redirect is not allowed'); - }); - - it('should return redirect url for providers logos', async () => { - jest.spyOn(apiConfigService, 'isMediaRedirectFeatureEnabled').mockReturnValueOnce(true); - - const uri = 'providers/asset/test.png'; - const expectedUri = 'keybase_processed_uploads/test.png'; - const expectedUrl = `https://s3.amazonaws.com/${expectedUri}`; - - const result = await mediaService.getRedirectUrl(uri); - - expect(result).toBe(expectedUrl); - }); - - it('should return redirect url for tokens logos', async () => { - jest.spyOn(apiConfigService, 'isMediaRedirectFeatureEnabled').mockReturnValueOnce(true); - - const network = 'devnet'; - jest.spyOn(apiConfigService, 'getNetwork').mockReturnValueOnce(network); - - const uri = 'tokens/asset/test.png'; - const expectedUri = `multiversx/mx-assets/master/${network}/tokens/test.png`; - const expectedUrl = `https://raw.githubusercontent.com/${expectedUri}`; - - const result = await mediaService.getRedirectUrl(uri); - - expect(result).toBe(expectedUrl); - }); - - it('should return redirect url for tokens logos on mainnet', async () => { - jest.spyOn(apiConfigService, 'isMediaRedirectFeatureEnabled').mockReturnValueOnce(true); - jest.spyOn(apiConfigService, 'getNetwork').mockReturnValueOnce('mainnet'); - - const uri = 'tokens/asset/test.png'; - const expectedUri = `multiversx/mx-assets/master/tokens/test.png`; - const expectedUrl = `https://raw.githubusercontent.com/${expectedUri}`; - - const result = await mediaService.getRedirectUrl(uri); - - expect(result).toBe(expectedUrl); - }); - - it('should return redirect url to storage urls', async () => { - jest.spyOn(apiConfigService, 'isMediaRedirectFeatureEnabled').mockReturnValueOnce(true); - jest.spyOn(apiConfigService, 'getMediaRedirectFileStorageUrls').mockReturnValueOnce(['https://s3.amazonaws.com']); - jest.spyOn(apiService, 'head').mockResolvedValueOnce({ status: 200 }); - - const uri = 'test.png'; - const expectedUrl = `https://s3.amazonaws.com/${uri}`; - - const result = await mediaService.getRedirectUrl(uri); - - expect(result).toBe(expectedUrl); - }); - - it('should return undefined when not found in file storage urls', async () => { - jest.spyOn(apiConfigService, 'isMediaRedirectFeatureEnabled').mockReturnValueOnce(true); - jest.spyOn(apiConfigService, 'getMediaRedirectFileStorageUrls').mockReturnValueOnce(['https://s3.amazonaws.com']); - jest.spyOn(apiService, 'head').mockResolvedValueOnce({ status: 404 }); - - const uri = 'test.png'; - - const result = await mediaService.getRedirectUrl(uri); - - expect(result).toBeUndefined(); - }); - - it('should return redirect url for nfts assets', async () => { - jest.spyOn(apiConfigService, 'isMediaRedirectFeatureEnabled').mockReturnValueOnce(true); - jest.spyOn(apiConfigService, 'getMediaRedirectFileStorageUrls').mockReturnValueOnce([]); - - const uri = 'nfts/asset/test.png'; - const expectedUri = `ipfs/test.png`; - const expectedUrl = `https://ipfs.io/${expectedUri}`; - - const result = await mediaService.getRedirectUrl(uri); - - expect(result).toBe(expectedUrl); - }); - - it('should return redirect to fallback thumbnail if not found in file storage urls', async () => { - jest.spyOn(apiConfigService, 'isMediaRedirectFeatureEnabled').mockReturnValueOnce(true); - jest.spyOn(apiConfigService, 'getMediaRedirectFileStorageUrls').mockReturnValueOnce(['https://s3.amazonaws.com']); - jest.spyOn(apiService, 'head').mockResolvedValueOnce({ status: 404 }); - - const uri = 'nfts/thumbnail/random'; - const expectedUrl = `https://s3.amazonaws.com/nfts/thumbnail/default.png`; - - const result = await mediaService.getRedirectUrl(uri); - - expect(result).toBe(expectedUrl); - }); - }); -}); diff --git a/src/test/unit/services/mex.token.charts.spec.ts b/src/test/unit/services/mex.token.charts.spec.ts index 2c41bc2cc..1f18810cf 100644 --- a/src/test/unit/services/mex.token.charts.spec.ts +++ b/src/test/unit/services/mex.token.charts.spec.ts @@ -94,7 +94,7 @@ describe('MexTokenChartsService', () => { jest.spyOn(mexTokenService, 'getMexTokenByIdentifier').mockResolvedValue(mockToken); jest.spyOn(mexTokenChartsService as any, 'isMexToken').mockReturnValue(true); - const result = await mexTokenChartsService.getTokenPricesDayResolutionRaw('TOKEN-123456', '1683561648'); + const result = await mexTokenChartsService.getTokenPricesDayResolutionRaw('TOKEN-123456'); if (result) { expect(result).toHaveLength(2); @@ -107,7 +107,7 @@ describe('MexTokenChartsService', () => { it('should return an empty array when no data is available', async () => { jest.spyOn(graphQlService, 'getExchangeServiceData').mockResolvedValue({}); jest.spyOn(mexTokenChartsService as any, 'isMexToken').mockReturnValue(true); - const result = await mexTokenChartsService.getTokenPricesDayResolutionRaw('TOKEN-123456', '1683561648'); + const result = await mexTokenChartsService.getTokenPricesDayResolutionRaw('TOKEN-123456'); expect(result).toEqual([]); }); diff --git a/src/test/unit/services/tokens.spec.ts b/src/test/unit/services/tokens.spec.ts index f025eaddb..064939059 100644 --- a/src/test/unit/services/tokens.spec.ts +++ b/src/test/unit/services/tokens.spec.ts @@ -31,6 +31,7 @@ import { NftCollection } from "src/endpoints/collections/entities/nft.collection import { EsdtSupply } from "src/endpoints/esdt/entities/esdt.supply"; import { TokenDetailed } from "src/endpoints/tokens/entities/token.detailed"; import { NumberUtils } from "@multiversx/sdk-nestjs-common"; +import { TokenAssetsPriceSourceType } from "../../../common/assets/entities/token.assets.price.source.type"; describe('Token Service', () => { let tokenService: TokenService; @@ -592,6 +593,25 @@ describe('Token Service', () => { expect(result).toBeGreaterThanOrEqual(261151384.6163954); expect(getAllTokensMock).toHaveBeenCalledTimes(1); }); + + it('should not include custom priced tokens in market cap', async () => { + const mockTokens = JSON.parse(fs.readFileSync(path.join(__dirname, '../../mocks/tokens.mock.json'), 'utf-8')); + const getAllTokensMock = jest.spyOn(tokenService, 'getAllTokens').mockResolvedValue(mockTokens); + + const result = await tokenService.getTokenMarketCapRaw(); + expect(result).toBeGreaterThanOrEqual(261151384.6163954); + expect(getAllTokensMock).toHaveBeenCalledTimes(1); + + const secondToken = mockTokens[1]; + secondToken.assets.priceSource = {type: 'customUrl'}; + const newExpectedMarketCap = result - secondToken.marketCap; + mockTokens[1] = secondToken; + + jest.spyOn(tokenService, 'getAllTokens').mockResolvedValue(mockTokens); + + const newResult = await tokenService.getTokenMarketCapRaw(); + expect(newResult).toBe(newExpectedMarketCap); + }); }); describe('getAllTokens', () => { @@ -787,6 +807,109 @@ describe('Token Service', () => { }); }); + it('adjusts the order depending on the price source and market cap', async () => { + jest.spyOn(tokenService['apiConfigService'], 'isTokensFetchFeatureEnabled').mockReturnValue(false); + jest.spyOn(tokenService['esdtService'], 'getAllFungibleTokenProperties').mockResolvedValue([ + new TokenProperties({ identifier: 'token1' }), + new TokenProperties({ identifier: 'token2' }), // <- will have custom price source + new TokenProperties({ identifier: 'token3' }), + new TokenProperties({ identifier: 'token4' }), + new TokenProperties({ identifier: 'token5' }), + ]); + + // Only token2 has a custom price source + // eslint-disable-next-line require-await + jest.spyOn(tokenService['assetsService'], 'getTokenAssets').mockImplementation(async (identifier: string) => { + if (identifier === 'token2') { + return new TokenAssets({ + name: `Token ${identifier}`, + priceSource: { + type: TokenAssetsPriceSourceType.customUrl, + path: '0.usdPrice', + url: 'url', + }, + }); + } + return new TokenAssets({ + name: `Token ${identifier}`, + // No priceSource + }); + }); + + jest.spyOn(tokenService['collectionService'], 'getNftCollections').mockResolvedValue([]); + + jest.spyOn(tokenService['dataApiService'], 'getEgldPrice').mockResolvedValue(0); + jest.spyOn(tokenService['dataApiService'], 'getEsdtTokenPrice').mockResolvedValue(1); + jest.spyOn(tokenService['esdtService'], 'getTokenSupply').mockResolvedValue({ + minted: '1000000', + initialMinted: '1000000', + burned: '0', + totalSupply: '1000000', + circulatingSupply: '1000000', + lockedAccounts: undefined, + }); + + // Fake other dependencies + jest.spyOn(tokenService as any, 'applyMexLiquidity').mockResolvedValue(undefined); + jest.spyOn(tokenService as any, 'applyMexPrices').mockResolvedValue(undefined); + jest.spyOn(tokenService as any, 'applyMexPairType').mockResolvedValue(undefined); + jest.spyOn(tokenService as any, 'applyMexPairTradesCount').mockResolvedValue(undefined); + jest.spyOn(tokenService['apiService'] as any, 'get').mockResolvedValue({data: [{usdPrice: 1.0}]}); + jest.spyOn(tokenService['cachingService'], 'batchApplyAll').mockImplementation( + // eslint-disable-next-line require-await + async (...args: unknown[]) => { + const tokens = args[0] as TokenDetailed[]; + const apply = args[3] as (token: TokenDetailed, assets: TokenAssets, fromGetter: boolean) => void; + + for (const token of tokens) { + if (token.identifier === 'token2') { + apply(token, new TokenAssets({ + name: `Token ${token.identifier}`, + priceSource: { + type: TokenAssetsPriceSourceType.customUrl, + path: '0.usdPrice', + url: 'url', + }, + }), true); + } else { + apply(token, new TokenAssets({ + name: `Token ${token.identifier}`, + // No priceSource + }), true); + } + } + } + ); + + // eslint-disable-next-line require-await + jest.spyOn(tokenService as any, 'batchProcessTokens').mockImplementation(async (tokens: any) => { + const marketCaps = { + token1: 500, + token2: 400, + token3: 300, + token4: 200, + token5: 100, + }; + for (const [index, token] of tokens.entries()) { + token.decimals = 18; + token.isLowLiquidity = false; + token.transactions = 10; + if (index === 3) { + continue; // make one of the tokens (token4) not to have any price or market cap at all + } + // @ts-ignore + token.marketCap = marketCaps[token.identifier]; + token.price = 1; + } + }); + + const result = await tokenService.getAllTokensRaw(); + const sortedIdentifiers = result.map(t => t.identifier); + + // token2 has custom price source, token4 does not have price/market cap at all + expect(sortedIdentifiers.slice(0, 5)).toEqual(['token5', 'token3', 'token1', 'token2', 'token4']); + }); + it('should return values from cache', async () => { const cachedValueMock = jest.spyOn(tokenService['cachingService'], 'getOrSet').mockResolvedValue(mockTokens); diff --git a/src/test/unit/utils/circuit.breaker.proxy.spec.ts b/src/test/unit/utils/circuit.breaker.proxy.spec.ts new file mode 100644 index 000000000..e92d88f1d --- /dev/null +++ b/src/test/unit/utils/circuit.breaker.proxy.spec.ts @@ -0,0 +1,316 @@ +import { ElasticService } from "@multiversx/sdk-nestjs-elastic"; +import { ApiConfigService } from "src/common/api-config/api.config.service"; +import { ElasticQuery } from "@multiversx/sdk-nestjs-elastic"; +import { EsCircuitBreakerProxy } from "../../../common/indexer/elastic/circuit-breaker/circuit.breaker.proxy.service"; + +describe('EsCircuitBreakerProxy', () => { + let proxy: EsCircuitBreakerProxy; + let mockElasticService: jest.Mocked; + let mockApiConfigService: jest.Mocked; + + const defaultConfig = { + durationThresholdMs: 1000, + failureCountThreshold: 2, + resetTimeoutMs: 2000, + }; + + beforeEach(() => { + mockElasticService = { + getCount: jest.fn(), + getList: jest.fn(), + getItem: jest.fn(), + getCustomValue: jest.fn(), + setCustomValue: jest.fn(), + setCustomValues: jest.fn(), + getScrollableList: jest.fn(), + get: jest.fn(), + post: jest.fn(), + } as any; + + mockApiConfigService = { + getElasticUrl: jest.fn().mockReturnValue('http://localhost:9200'), + isElasticCircuitBreakerEnabled: jest.fn().mockReturnValue(true), + getElasticCircuitBreakerConfig: jest.fn().mockReturnValue(defaultConfig), + } as any; + + proxy = new EsCircuitBreakerProxy(mockApiConfigService, mockElasticService); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + describe('Circuit Breaker State Management', () => { + it('should start with closed circuit', () => { + expect(proxy['isCircuitOpen']).toBe(false); + expect(proxy['failureCount']).toBe(0); + }); + + it('should open circuit after reaching failure threshold', async () => { + const failureThreshold = 2; + mockApiConfigService.getElasticCircuitBreakerConfig.mockReturnValue({ + ...defaultConfig, + failureCountThreshold: failureThreshold, + }); + + mockElasticService.getCount.mockRejectedValue(new Error('Service unavailable')); + + for (let i = 0; i < failureThreshold; i++) { + try { + await proxy.getCount('test', new ElasticQuery()); + } catch (error) { + // Expected error + } + } + + expect(proxy['isCircuitOpen']).toBe(true); + expect(proxy['failureCount']).toBe(failureThreshold); + + await expect(proxy.getCount('test', new ElasticQuery())) + .rejects + .toThrow('Service Unavailable'); + }); + + it('should reject requests when circuit is open', async () => { + proxy['isCircuitOpen'] = true; + proxy['lastFailureTime'] = Date.now(); + + await expect(proxy.getCount('test', new ElasticQuery())) + .rejects + .toThrow('Service Unavailable'); + }); + + it('should attempt to reset circuit after reset timeout', async () => { + const resetTimeout = 1000; + mockApiConfigService.getElasticCircuitBreakerConfig.mockReturnValue({ + ...defaultConfig, + resetTimeoutMs: resetTimeout, + }); + + // Force circuit to open and set last failure time to be older than reset timeout + proxy['isCircuitOpen'] = true; + proxy['lastFailureTime'] = Date.now() - (resetTimeout + 1000); + proxy['failureCount'] = 2; // Set failure count to simulate previous failures + + // Mock a successful response + mockElasticService.getCount.mockResolvedValue(10); + + // The circuit should be reset and the request should succeed + const result = await proxy.getCount('test', new ElasticQuery()); + expect(result).toBe(10); + expect(proxy['isCircuitOpen']).toBe(false); + expect(proxy['failureCount']).toBe(0); + }); + }); + + describe('Timeout Handling', () => { + it('should timeout requests that take too long', async () => { + const timeout = 500; + mockApiConfigService.getElasticCircuitBreakerConfig.mockReturnValue({ + ...defaultConfig, + durationThresholdMs: timeout, + }); + + mockElasticService.getCount.mockImplementation(() => + new Promise(() => { }) + ); + + await expect(Promise.race([ + proxy.getCount('test', new ElasticQuery()), + new Promise((_, reject) => setTimeout(() => reject(new Error('Operation timed out')), timeout + 100)), + ])).rejects.toThrow('Operation timed out'); + }); + + it('should complete requests within timeout', async () => { + const timeout = 500; + mockApiConfigService.getElasticCircuitBreakerConfig.mockReturnValue({ + ...defaultConfig, + durationThresholdMs: timeout, + }); + + mockElasticService.getCount.mockImplementation(() => + new Promise(resolve => setTimeout(() => resolve(10), timeout - 100)) + ); + + const result = await proxy.getCount('test', new ElasticQuery()); + expect(result).toBe(10); + }); + }); + + describe('Method Proxying', () => { + it('should proxy getCount calls', async () => { + mockElasticService.getCount.mockResolvedValue(10); + const query = new ElasticQuery(); + + const result = await proxy.getCount('test', query); + expect(result).toBe(10); + expect(mockElasticService.getCount).toHaveBeenCalledWith('test', query); + }); + + it('should proxy getList calls', async () => { + const mockData = [{ id: 1 }, { id: 2 }]; + mockElasticService.getList.mockResolvedValue(mockData); + const query = new ElasticQuery(); + + const result = await proxy.getList('test', 'id', query); + expect(result).toEqual(mockData); + expect(mockElasticService.getList).toHaveBeenCalledWith('test', 'id', query); + }); + + it('should proxy getItem calls', async () => { + const mockItem = { id: 1, name: 'test' }; + mockElasticService.getItem.mockResolvedValue(mockItem); + + const result = await proxy.getItem('test', 'id', '1'); + expect(result).toEqual(mockItem); + expect(mockElasticService.getItem).toHaveBeenCalledWith('test', 'id', '1'); + }); + + it('should proxy getCustomValue calls', async () => { + const mockValue = { key: 'value' }; + mockElasticService.getCustomValue.mockResolvedValue(mockValue); + + const result = await proxy.getCustomValue('test', 'id', 'key'); + expect(result).toEqual(mockValue); + expect(mockElasticService.getCustomValue).toHaveBeenCalledWith('test', 'id', 'key'); + }); + + it('should proxy setCustomValue calls', async () => { + mockElasticService.setCustomValue.mockResolvedValue(undefined); + + await proxy.setCustomValue('test', 'id', 'key', 'value'); + expect(mockElasticService.setCustomValue).toHaveBeenCalledWith('test', 'id', 'key', 'value'); + }); + + it('should proxy setCustomValues calls', async () => { + mockElasticService.setCustomValues.mockResolvedValue(undefined); + const values = { key1: 'value1', key2: 'value2' }; + + await proxy.setCustomValues('test', 'id', values); + expect(mockElasticService.setCustomValues).toHaveBeenCalledWith('test', 'id', values); + }); + + it('should proxy getScrollableList calls', async () => { + const mockAction = jest.fn(); + mockElasticService.getScrollableList.mockResolvedValue(undefined); + const query = new ElasticQuery(); + + await proxy.getScrollableList('test', 'id', query, mockAction); + expect(mockElasticService.getScrollableList).toHaveBeenCalledWith('test', 'id', query, mockAction); + }); + + it('should proxy get calls', async () => { + const mockResponse = { data: 'test' }; + mockElasticService.get.mockResolvedValue(mockResponse); + + const result = await proxy.get('test'); + expect(result).toEqual(mockResponse); + expect(mockElasticService.get).toHaveBeenCalledWith('test'); + }); + + it('should proxy post calls', async () => { + const mockResponse = { data: 'test' }; + mockElasticService.post.mockResolvedValue(mockResponse); + const data = { key: 'value' }; + + const result = await proxy.post('test', data); + expect(result).toEqual(mockResponse); + expect(mockElasticService.post).toHaveBeenCalledWith('test', data); + }); + }); + + describe('Error Handling', () => { + it('should handle and propagate errors', async () => { + const error = new Error('Test error'); + mockElasticService.getCount.mockRejectedValue(error); + + await expect(proxy.getCount('test', new ElasticQuery())) + .rejects + .toThrow('Service Unavailable'); + }); + + it('should increment failure count on errors', async () => { + mockElasticService.getCount.mockRejectedValue(new Error('Test error')); + + try { + await proxy.getCount('test', new ElasticQuery()); + } catch (error) { + // Expected error + } + + expect(proxy['failureCount']).toBe(1); + }); + + it('should reset failure count on successful request', async () => { + mockElasticService.getCount.mockRejectedValueOnce(new Error('Test error')); + try { + await proxy.getCount('test', new ElasticQuery()); + } catch (error) { + // Expected error + } + + mockElasticService.getCount.mockResolvedValueOnce(10); + await proxy.getCount('test', new ElasticQuery()); + + expect(proxy['failureCount']).toBe(0); + }); + }); + + describe('Configuration Updates', () => { + it('should use updated timeout value', async () => { + const newTimeout = 500; + mockApiConfigService.getElasticCircuitBreakerConfig.mockReturnValue({ + ...defaultConfig, + durationThresholdMs: newTimeout, + }); + + mockElasticService.getCount.mockImplementation(() => + new Promise(resolve => setTimeout(() => resolve(10), newTimeout - 100)) + ); + + const result = await proxy.getCount('test', new ElasticQuery()); + expect(result).toBe(10); + }); + + it('should use updated failure threshold', async () => { + const newThreshold = 1; + mockApiConfigService.getElasticCircuitBreakerConfig.mockReturnValue({ + ...defaultConfig, + failureCountThreshold: newThreshold, + }); + mockElasticService.getCount.mockRejectedValue(new Error('Service unavailable')); + + for (let i = 0; i < newThreshold; i++) { + try { + await proxy.getCount('test', new ElasticQuery()); + } catch (error) { + // Expected error + } + } + + expect(proxy['failureCount']).toBe(newThreshold); + }); + + it('should use updated reset timeout', async () => { + const newResetTimeout = 1000; + mockApiConfigService.getElasticCircuitBreakerConfig.mockReturnValue({ + ...defaultConfig, + resetTimeoutMs: newResetTimeout, + }); + + // Force circuit to open and set last failure time to be older than reset timeout + proxy['isCircuitOpen'] = true; + proxy['lastFailureTime'] = Date.now() - (newResetTimeout + 1000); + proxy['failureCount'] = 2; // Set failure count to simulate previous failures + + // Mock a successful response + mockElasticService.getCount.mockResolvedValue(10); + + // The circuit should be reset and the request should succeed + const result = await proxy.getCount('test', new ElasticQuery()); + expect(result).toBe(10); + expect(proxy['isCircuitOpen']).toBe(false); + expect(proxy['failureCount']).toBe(0); + }); + }); +}); diff --git a/src/utils/cache.info.ts b/src/utils/cache.info.ts index 531400971..6ca7e69ac 100644 --- a/src/utils/cache.info.ts +++ b/src/utils/cache.info.ts @@ -162,9 +162,9 @@ export class CacheInfo { }; } - static TokenDailyChart(tokenIdentifier: string, after: string): CacheInfo { + static TokenDailyChart(tokenIdentifier: string): CacheInfo { return { - key: `tokenDailyChart:${tokenIdentifier}:${after}`, + key: `tokenDailyChart:${tokenIdentifier}`, ttl: Constants.oneDay(), }; } diff --git a/src/utils/token.helpers.ts b/src/utils/token.helpers.ts index 7ed8d6091..c88211a4c 100644 --- a/src/utils/token.helpers.ts +++ b/src/utils/token.helpers.ts @@ -18,6 +18,12 @@ export class TokenHelpers { uri = ApiUtils.replaceUri(uri, 'https://gateway.pinata.cloud/ipfs', prefix); uri = ApiUtils.replaceUri(uri, 'https://dweb.link/ipfs', prefix); uri = ApiUtils.replaceUri(uri, 'ipfs:/', prefix); + uri = ApiUtils.replaceUri(uri, 'https://media.elrond.com/nfts/asset', prefix); + uri = ApiUtils.replaceUri(uri, 'https://devnet-media.elrond.com/nfts/asset', prefix); + uri = ApiUtils.replaceUri(uri, 'https://testnet-media.elrond.com/nfts/asset', prefix); + uri = ApiUtils.replaceUri(uri, 'https://api.multiversx.com/media/nfts/asset', prefix); + uri = ApiUtils.replaceUri(uri, 'https://devnet-api.multiversx.com/media/nfts/asset', prefix); + uri = ApiUtils.replaceUri(uri, 'https://testnet-api.multiversx.com/media/nfts/asset', prefix); uri = uri.replace(/https\:\/\/\w*\.mypinata\.cloud\/ipfs/, prefix); if (uri.endsWith('.ipfs.dweb.link')) {