diff --git a/src/common/indexer/entities/collection.ts b/src/common/indexer/entities/collection.ts index e6a8f12dc..5190e17f6 100644 --- a/src/common/indexer/entities/collection.ts +++ b/src/common/indexer/entities/collection.ts @@ -1,3 +1,16 @@ +export interface CollectionProperties { + canMint?: boolean; + canBurn?: boolean; + canUpgrade?: boolean; + canTransferNFTCreateRole?: boolean; + canAddSpecialRoles?: boolean; + canPause?: boolean; + canFreeze?: boolean; + canWipe?: boolean; + canChangeOwner?: boolean; + canCreateMultiShard?: boolean; +} + export interface Collection { _id: string; name: string; @@ -5,9 +18,11 @@ export interface Collection { token: string; issuer: string; currentOwner: string; + numDecimals?: number; type: string; timestamp: number; ownersHistory: { address: string, timestamp: number }[]; + properties?: CollectionProperties; api_isVerified?: boolean; api_nftCount?: number; api_holderCount?: number; diff --git a/src/endpoints/accounts/account.service.ts b/src/endpoints/accounts/account.service.ts index 104a1b5be..5724cee62 100644 --- a/src/endpoints/accounts/account.service.ts +++ b/src/endpoints/accounts/account.service.ts @@ -22,6 +22,7 @@ import { GatewayService } from 'src/common/gateway/gateway.service'; import { IndexerService } from "src/common/indexer/indexer.service"; import { AccountAssets } from 'src/common/assets/entities/account.assets'; import { CacheInfo } from 'src/utils/cache.info'; +import { ConcurrencyUtils } from 'src/utils/concurrency.utils'; import { UsernameService } from '../usernames/username.service'; import { ContractUpgrades } from './entities/contract.upgrades'; import { AccountVerification } from './entities/account.verification'; @@ -356,36 +357,41 @@ export class AccountService { } } - for (const account of accounts) { - account.shard = AddressUtils.computeShard(AddressUtils.bech32Decode(account.address), shardCount); - account.assets = assets[account.address]; + await ConcurrencyUtils.executeWithConcurrencyLimit( + accounts, + async (account) => { + account.shard = AddressUtils.computeShard(AddressUtils.bech32Decode(account.address), shardCount); + account.assets = assets[account.address]; - if (options.withDeployInfo && AddressUtils.isSmartContractAddress(account.address)) { - const [deployedAt, deployTxHash] = await Promise.all([ - this.getAccountDeployedAt(account.address), - this.getAccountDeployedTxHash(account.address), - ]); + if (options.withDeployInfo && AddressUtils.isSmartContractAddress(account.address)) { + const [deployedAt, deployTxHash] = await Promise.all([ + this.getAccountDeployedAt(account.address), + this.getAccountDeployedTxHash(account.address), + ]); - account.deployedAt = deployedAt; - account.deployTxHash = deployTxHash; - } + account.deployedAt = deployedAt; + account.deployTxHash = deployTxHash; + } - if (options.withTxCount) { - account.txCount = await this.getAccountTxCount(account.address); - } + if (options.withTxCount) { + account.txCount = await this.getAccountTxCount(account.address); + } - if (options.withScrCount) { - account.scrCount = await this.getAccountScResults(account.address); - } + if (options.withScrCount) { + account.scrCount = await this.getAccountScResults(account.address); + } - if (options.withOwnerAssets && account.ownerAddress) { - account.ownerAssets = assets[account.ownerAddress]; - } + if (options.withOwnerAssets && account.ownerAddress) { + account.ownerAssets = assets[account.ownerAddress]; + } - if (verifiedAccounts && verifiedAccounts.includes(account.address)) { - account.isVerified = true; - } - } + if (verifiedAccounts && verifiedAccounts.includes(account.address)) { + account.isVerified = true; + } + }, + 6, + 'AccountService.getAccountsRaw', + ); return accounts; } diff --git a/src/endpoints/collections/collection.service.ts b/src/endpoints/collections/collection.service.ts index ce8378053..11cbc2098 100644 --- a/src/endpoints/collections/collection.service.ts +++ b/src/endpoints/collections/collection.service.ts @@ -29,6 +29,7 @@ import { CollectionLogo } from "./entities/collection.logo"; import { ScamInfo } from "src/common/entities/scam-info.dto"; import { NftType as ElasticNftType } from "src/common/indexer/entities/nft.type"; import { NftSubType } from "../nfts/entities/nft.sub.type"; +import { ConcurrencyUtils } from "src/utils/concurrency.utils"; @Injectable() export class CollectionService { @@ -50,10 +51,42 @@ export class CollectionService { } async getNftCollections(pagination: QueryPagination, filter: CollectionFilter): Promise { + if (this.isCacheableCollectionFilter(filter)) { + return await this.cachingService.getOrSet( + CacheInfo.Collections(pagination).key, + () => this.fetchAndProcessCollections(pagination, filter), + CacheInfo.Collections(pagination).ttl, + ); + } + + return await this.fetchAndProcessCollections(pagination, filter); + } + + private async fetchAndProcessCollections(pagination: QueryPagination, filter: CollectionFilter): Promise { const tokenCollections = await this.indexerService.getNftCollections(pagination, filter); return await this.processNftCollections(tokenCollections); } + private isCacheableCollectionFilter(filter: CollectionFilter): boolean { + return !filter.collection && + !(filter.identifiers && filter.identifiers.length > 0) && + !(filter.type && filter.type.length > 0) && + !(filter.subType && filter.subType.length > 0) && + !filter.search && + !filter.owner && + filter.before === undefined && + filter.after === undefined && + filter.canCreate === undefined && + filter.canBurn === undefined && + filter.canAddQuantity === undefined && + filter.canUpdateAttributes === undefined && + filter.canAddUri === undefined && + filter.canTransferRole === undefined && + filter.excludeMetaESDT === undefined && + filter.sort === undefined && + filter.order === undefined; + } + async getNftCollectionsByIds(identifiers: Array): Promise { const tokenCollections = await this.indexerService.getNftCollectionsByIds(identifiers); return await this.processNftCollections(tokenCollections); @@ -62,18 +95,41 @@ export class CollectionService { private async processNftCollections(tokenCollections: Collection[]): Promise { const collectionsIdentifiers = tokenCollections.map((collection) => collection.token); - const indexedCollections = new Map(); - for (const collection of tokenCollections) { - indexedCollections.set(collection.token, collection); - } + const collectionsAssets = await this.batchGetCollectionsAssets(collectionsIdentifiers); - const nftCollections: NftCollection[] = await this.applyPropertiesToCollections(collectionsIdentifiers); + const nftCollections: NftCollection[] = []; - for (const nftCollection of nftCollections) { - const indexedCollection = indexedCollections.get(nftCollection.collection); - if (indexedCollection) { - this.applyPropertiesToCollectionFromElasticSearch(nftCollection, indexedCollection); - } + for (const esCollection of tokenCollections) { + const identifierParts = esCollection.token.split('-'); + const ticker = identifierParts[0]; + const collectionBase = identifierParts.slice(0, 2).join('-'); + const assets = collectionsAssets[esCollection.token]; + const props = esCollection.properties; + + const isMetaESDT = esCollection.type === ElasticNftType.MetaESDT || esCollection.type === ElasticNftType.DynamicMetaESDT; + + const nftCollection = new NftCollection({ + name: esCollection.name, + collection: collectionBase, + ticker: ticker, + owner: esCollection.currentOwner, + assets: assets, + canFreeze: props?.canFreeze, + canWipe: props?.canWipe, + canPause: props?.canPause, + canTransferNftCreateRole: props?.canTransferNFTCreateRole, + canChangeOwner: props?.canChangeOwner, + canUpgrade: props?.canUpgrade, + canAddSpecialRoles: props?.canAddSpecialRoles, + decimals: isMetaESDT ? esCollection.numDecimals : undefined, + }); + + nftCollection.ticker = nftCollection.assets ? ticker : nftCollection.collection; + + // Apply additional ES fields (type, timestamp, counts, scamInfo) + this.applyPropertiesToCollectionFromElasticSearch(nftCollection, esCollection); + + nftCollections.push(nftCollection); } return nftCollections; @@ -113,6 +169,10 @@ export class CollectionService { } } + async buildCollectionsFromElasticData(esCollections: Collection[]): Promise { + return await this.processNftCollections(esCollections); + } + async applyPropertiesToCollections(collectionsIdentifiers: string[]): Promise { const nftCollections: NftCollection[] = []; @@ -159,30 +219,60 @@ export class CollectionService { } async batchGetCollectionsProperties(identifiers: string[]): Promise<{ [key: string]: TokenProperties | undefined }> { - if (this.apiConfigService.getCollectionPropertiesFromGateway()) { - return await this.getCollectionProperties(identifiers); + const result: { [key: string]: TokenProperties | undefined } = {}; + const chunks = this.splitIntoChunks(identifiers, 300); + + const chunkResults = await ConcurrencyUtils.executeWithConcurrencyLimit( + chunks, + async (chunk) => { + if (this.apiConfigService.getCollectionPropertiesFromGateway()) { + return await this.getCollectionProperties(chunk); + } + return await this.getEsdtProperties(chunk); + }, + 4, + 'CollectionService.batchGetCollectionsProperties' + ); + + for (const chunkResult of chunkResults) { + Object.assign(result, chunkResult); } - return await this.getEsdtProperties(identifiers); + return result; } async batchGetCollectionsAssets(identifiers: string[]): Promise<{ [key: string]: TokenAssets | undefined }> { const collectionsAssets: { [key: string]: TokenAssets | undefined } = {}; - const allAssets = await this.assetsService.getAllTokenAssets(); - - await this.cachingService.batchApplyAll( - identifiers, - identifier => CacheInfo.EsdtAssets(identifier).key, - identifier => Promise.resolve(allAssets[identifier]), - (identifier, properties) => collectionsAssets[identifier] = properties, - CacheInfo.EsdtAssets('').ttl + const chunks = this.splitIntoChunks(identifiers, 300); + + await ConcurrencyUtils.executeWithConcurrencyLimit( + chunks, + async (chunk) => { + await this.cachingService.batchApplyAll( + chunk, + identifier => CacheInfo.EsdtAssets(identifier).key, + identifier => Promise.resolve(allAssets[identifier]), + (identifier, properties) => collectionsAssets[identifier] = properties, + CacheInfo.EsdtAssets('').ttl + ); + }, + 4, + 'CollectionService.batchGetCollectionsAssets' ); return collectionsAssets; } async getNftCollectionCount(filter: CollectionFilter): Promise { + if (this.isCacheableCollectionFilter(filter)) { + return await this.cachingService.getOrSet( + CacheInfo.CollectionsCount.key, + async () => await this.indexerService.getNftCollectionCount(filter), + CacheInfo.CollectionsCount.ttl, + ); + } + return await this.indexerService.getNftCollectionCount(filter); } @@ -201,7 +291,11 @@ export class CollectionService { return undefined; } - return await this.assetsService.getCollectionRanks(identifier); + return await this.cachingService.getOrSet( + CacheInfo.CollectionRanksForIdentifier(identifier).key, + async () => await this.assetsService.getCollectionRanks(identifier), + CacheInfo.CollectionRanksForIdentifier(identifier).ttl, + ); } async getNftCollection(identifier: string): Promise { @@ -218,19 +312,15 @@ export class CollectionService { return undefined; } - const [collection] = await this.applyPropertiesToCollections([identifier]); + const [collection] = await this.buildCollectionsFromElasticData([elasticCollection]); if (!collection) { return undefined; } const collectionDetailed = ApiUtils.mergeObjects(new NftCollectionDetailed(), collection); - collectionDetailed.type = elasticCollection.type as NftType; - collectionDetailed.timestamp = elasticCollection.timestamp; - this.applyPropertiesToCollectionFromElasticSearch(collectionDetailed, elasticCollection); - - collectionDetailed.traits = await this.persistenceService.getCollectionTraits(identifier) ?? []; + collectionDetailed.traits = await this.getCollectionTraitsCached(identifier); await this.applyCollectionRoles(collectionDetailed, elasticCollection); @@ -238,7 +328,7 @@ export class CollectionService { } async applyCollectionRoles(collection: NftCollectionDetailed | TokenDetailed, elasticCollection: any) { - collection.roles = await this.getNftCollectionRolesFromGateway(elasticCollection); + collection.roles = await this.getCollectionRolesCached(elasticCollection.token, elasticCollection); const isTransferProhibitedByDefault = collection.roles?.some(x => x.canTransfer === true) === true; collection.canTransfer = !isTransferProhibitedByDefault; if (collection.canTransfer) { @@ -330,8 +420,19 @@ export class CollectionService { } async getCollectionCountForAddress(address: string, filter: CollectionFilter): Promise { - const collections = await this.getCollectionsForAddress(address, filter, new QueryPagination({ from: 0, size: 10000 })); + if (this.isDefaultAddressCollectionFilter(filter)) { + return await this.cachingService.getOrSet( + CacheInfo.CollectionCountForAddress(address).key, + () => this.computeCollectionCountForAddress(address, filter), + CacheInfo.CollectionCountForAddress(address).ttl, + ); + } + + return await this.computeCollectionCountForAddress(address, filter); + } + private async computeCollectionCountForAddress(address: string, filter: CollectionFilter): Promise { + const collections = await this.getCollectionsForAddress(address, filter, new QueryPagination({ from: 0, size: 10000 })); return collections.length; } @@ -351,25 +452,63 @@ export class CollectionService { } async getCollectionsForAddress(address: string, filter: CollectionFilter, pagination: QueryPagination): Promise { + if (this.isDefaultAddressCollectionFilter(filter)) { + const cacheInfo = CacheInfo.CollectionsForAddress(address, pagination); + return await this.cachingService.getOrSet( + cacheInfo.key, + () => this.fetchCollectionsForAddress(address, filter, pagination), + cacheInfo.ttl, + ); + } + + return await this.fetchCollectionsForAddress(address, filter, pagination); + } + + private async fetchCollectionsForAddress(address: string, filter: CollectionFilter, pagination: QueryPagination): Promise { const collectionsRaw = await this.indexerService.getCollectionsForAddress(address, filter, pagination); - const collections = await this.getNftCollections( - new QueryPagination({ from: 0, size: collectionsRaw.length }), - new CollectionFilter({ identifiers: collectionsRaw.map((x: any) => x.collection) }) - ); - const accountCollections = collections.map(collection => ApiUtils.mergeObjects(new NftCollectionAccount(), collection)); + if (collectionsRaw.length === 0) { + return []; + } - for (const collection of accountCollections) { - const item = collectionsRaw.find(x => x.collection === collection.collection); - if (item) { - collection.count = item.count; + const identifiers = collectionsRaw.map((x: any) => x.collection); + const collections = await this.getNftCollectionsByIds(identifiers); + + const collectionMap = new Map(); + for (const collection of collections) { + collectionMap.set(collection.collection, collection); + } + + const accountCollections: NftCollectionAccount[] = []; + for (const raw of collectionsRaw) { + const collection = collectionMap.get(raw.collection); + if (collection) { + const accountCollection = ApiUtils.mergeObjects(new NftCollectionAccount(), collection); + accountCollection.count = raw.count; + accountCollections.push(accountCollection); } } return accountCollections; } + private isDefaultAddressCollectionFilter(filter: CollectionFilter): boolean { + return !filter.search && + !(filter.type && filter.type.length > 0) && + !(filter.subType && filter.subType.length > 0) && + !filter.excludeMetaESDT && + !filter.collection; + } + async getCollectionCountForAddressWithRoles(address: string, filter: CollectionFilter): Promise { + if (this.isDefaultAddressCollectionFilter(filter)) { + return await this.cachingService.getOrSet( + CacheInfo.CollectionRolesCountForAddress(address).key, + () => this.esdtAddressService.getCollectionCountForAddressFromElastic(address, filter), + CacheInfo.CollectionRolesCountForAddress(address).ttl, + ); + } + return await this.esdtAddressService.getCollectionCountForAddressFromElastic(address, filter); } @@ -383,7 +522,7 @@ export class CollectionService { } async getLogoPng(identifier: string): Promise { - const collectionLogo = await this.getCollectionLogo(identifier); + const collectionLogo = await this.getCollectionLogoCached(identifier); if (!collectionLogo) { return; } @@ -392,7 +531,7 @@ export class CollectionService { } async getLogoSvg(identifier: string): Promise { - const collectionLogo = await this.getCollectionLogo(identifier); + const collectionLogo = await this.getCollectionLogoCached(identifier); if (!collectionLogo) { return; } @@ -427,4 +566,40 @@ export class CollectionService { return collectionsProperties; } + + private splitIntoChunks(items: T[], chunkSize: number): T[][] { + if (chunkSize <= 0) { + return [items]; + } + + const chunks: T[][] = []; + for (let i = 0; i < items.length; i += chunkSize) { + chunks.push(items.slice(i, i + chunkSize)); + } + return chunks; + } + + private async getCollectionTraitsCached(identifier: string): Promise { + return await this.cachingService.getOrSet( + CacheInfo.CollectionTraits(identifier).key, + async () => await this.persistenceService.getCollectionTraits(identifier) ?? [], + CacheInfo.CollectionTraits(identifier).ttl, + ); + } + + private async getCollectionRolesCached(identifier: string, elasticCollection: any): Promise { + return await this.cachingService.getOrSet( + CacheInfo.CollectionRoles(identifier).key, + async () => await this.getNftCollectionRolesFromGateway(elasticCollection), + CacheInfo.CollectionRoles(identifier).ttl, + ); + } + + private async getCollectionLogoCached(identifier: string): Promise { + return await this.cachingService.getOrSet( + CacheInfo.CollectionLogo(identifier).key, + async () => await this.getCollectionLogo(identifier), + CacheInfo.CollectionLogo(identifier).ttl, + ); + } } diff --git a/src/endpoints/esdt/esdt.address.service.ts b/src/endpoints/esdt/esdt.address.service.ts index 34ea3dc94..c0b751505 100644 --- a/src/endpoints/esdt/esdt.address.service.ts +++ b/src/endpoints/esdt/esdt.address.service.ts @@ -101,14 +101,13 @@ export class EsdtAddressService { async getCollectionsForAddress(address: string, filter: CollectionFilter, pagination: QueryPagination): Promise { const tokenCollections = await this.indexerService.getNftCollections(pagination, filter, address); - const collectionsIdentifiers = tokenCollections.map((collection) => collection.token); const indexedCollections: Record = {}; for (const collection of tokenCollections) { indexedCollections[collection.token] = collection; } - const accountCollections = await this.collectionService.applyPropertiesToCollections(collectionsIdentifiers); + const accountCollections = await this.collectionService.buildCollectionsFromElasticData(tokenCollections); const collectionsWithRoles: NftCollectionWithRoles[] = []; diff --git a/src/endpoints/nfts/nft.service.ts b/src/endpoints/nfts/nft.service.ts index a9e8a3416..6c2f0cf9d 100644 --- a/src/endpoints/nfts/nft.service.ts +++ b/src/endpoints/nfts/nft.service.ts @@ -31,6 +31,7 @@ import { SortCollectionNfts } from "../collections/entities/sort.collection.nfts import { TokenAssets } from "src/common/assets/entities/token.assets"; import { ScamInfo } from "src/common/entities/scam-info.dto"; import { NftSubType } from "./entities/nft.sub.type"; +import { ConcurrencyUtils } from "src/utils/concurrency.utils"; @Injectable() export class NftService { @@ -65,6 +66,19 @@ export class NftService { } async getNfts(queryPagination: QueryPagination, filter: NftFilter, queryOptions?: NftQueryOptions): Promise { + if (this.isCacheableNftList(filter, queryOptions)) { + const cacheInfo = CacheInfo.Nfts(queryPagination); + return await this.cachingService.getOrSet( + cacheInfo.key, + () => this.fetchAndProcessNfts(queryPagination, filter, queryOptions), + cacheInfo.ttl, + ); + } + + return await this.fetchAndProcessNfts(queryPagination, filter, queryOptions); + } + + private async fetchAndProcessNfts(queryPagination: QueryPagination, filter: NftFilter, queryOptions?: NftQueryOptions): Promise { const { from, size } = queryPagination; const nfts = await this.getNftsInternal({ from, size }, filter); @@ -114,21 +128,15 @@ export class NftService { return; } - const nftsIdentifiers = nfts.filter(x => x.type === NftType.NonFungibleESDT).map(x => x.identifier); + const nftsIdentifiers = nfts + .filter(x => x.type === NftType.NonFungibleESDT) + .map(x => x.identifier); if (nftsIdentifiers.length === 0) { return; } - const accountsEsdts = await this.getAccountEsdtByIdentifiers(nftsIdentifiers, { - from: 0, - size: nftsIdentifiers.length, - }); - - const ownerMap = accountsEsdts.reduce((acc: Record, accountEsdt: any) => { - acc[accountEsdt.identifier] = accountEsdt.address; - return acc; - }, {}); + const ownerMap = await this.getOwnersBulk(nftsIdentifiers); for (const nft of nfts) { if (nft.type === NftType.NonFungibleESDT && ownerMap[nft.identifier]) { @@ -182,6 +190,29 @@ export class NftService { ); } + private async getOwnersBulk(identifiers: string[], chunkSize: number = 512, concurrencyLimit: number = 4): Promise> { + if (identifiers.length === 0) { + return {}; + } + + const chunks = BatchUtils.splitArrayIntoChunks(identifiers.distinct(), chunkSize); + const results = await ConcurrencyUtils.executeWithConcurrencyLimit( + chunks, + async (chunk) => await this.getAccountEsdtByIdentifiers(chunk, { from: 0, size: chunk.length }), + concurrencyLimit, + 'NftService.getOwnersBulk' + ); + + const ownerMap: Record = {}; + for (const chunkResult of results) { + for (const accountEsdt of chunkResult ?? []) { + ownerMap[accountEsdt.identifier] = accountEsdt.address; + } + } + + return ownerMap; + } + private async batchApplyMedia(nfts: Nft[], fields?: string[]) { if (fields && !fields.includes('media')) { return; @@ -509,15 +540,24 @@ export class NftService { } async getNftCount(filter: NftFilter): Promise { + if (this.isCacheableNftCount(filter)) { + return await this.cachingService.getOrSet( + CacheInfo.NftsCount.key, + async () => await this.indexerService.getNftCount(filter), + CacheInfo.NftsCount.ttl, + ); + } + return await this.indexerService.getNftCount(filter); } async getNftsForAddress(address: string, queryPagination: QueryPagination, filter: NftFilter, fields?: string[], queryOptions?: NftQueryOptions, source?: EsdtDataSource): Promise { let nfts = await this.esdtAddressService.getNftsForAddress(address, filter, queryPagination, source, queryOptions); - for (const nft of nfts) { + + await Promise.all(nfts.map(async (nft) => { await this.applyAssetsAndTicker(nft, fields); await this.applyPriceUsd(nft, fields); - } + })); if (queryOptions && queryOptions.withSupply) { const supplyNfts = nfts.filter(nft => nft.type.in(NftType.SemiFungibleESDT, NftType.MetaESDT)); @@ -549,23 +589,24 @@ export class NftService { nfts = this.applyScamFilter(nfts, filter); - for (const nft of nfts) { - await this.applyUnlockFields(nft, fields); - } + await Promise.all(nfts.map(nft => this.applyUnlockFields(nft, fields))); return nfts; } private async getNftsInternalByIdentifiers(identifiers: string[]): Promise { - const chunks = BatchUtils.splitArrayIntoChunks(identifiers, 1024); - const result: Nft[] = []; - for (const identifiers of chunks) { - const internalNfts = await this.getNftsInternal(new QueryPagination({ from: 0, size: identifiers.length }), new NftFilter({ identifiers })); - - result.push(...internalNfts); - } + const chunks = BatchUtils.splitArrayIntoChunks(identifiers, 512); + const results = await ConcurrencyUtils.executeWithConcurrencyLimit( + chunks, + async (chunk) => await this.getNftsInternal( + new QueryPagination({ from: 0, size: chunk.length }), + new NftFilter({ identifiers: chunk }) + ), + 4, + 'NftService.getNftsInternalByIdentifiers' + ); - return result; + return results.flat(); } private async applyPriceUsd(nft: NftAccount, fields?: string[]) { @@ -738,4 +779,39 @@ export class NftService { this.logger.error(error); } } + + private isDefaultNftFilter(filter: NftFilter): boolean { + return !filter.search && + !(filter.identifiers && filter.identifiers.length > 0) && + !(filter.type && filter.type.length > 0) && + !(filter.subType && filter.subType.length > 0) && + !filter.collection && + !(filter.collections && filter.collections.length > 0) && + !(filter.tags && filter.tags.length > 0) && + !filter.name && + !filter.creator && + filter.hasUris === undefined && + filter.includeFlagged === undefined && + filter.before === undefined && + filter.after === undefined && + filter.nonceBefore === undefined && + filter.nonceAfter === undefined && + filter.isWhitelistedStorage === undefined && + filter.isNsfw === undefined && + filter.isScam === undefined && + filter.scamType === undefined && + !filter.traits && + filter.excludeMetaESDT === undefined && + filter.sort === undefined && + filter.order === undefined; + } + + private isCacheableNftList(filter: NftFilter, queryOptions?: NftQueryOptions): boolean { + const hasHeavyOptions = queryOptions?.withOwner || queryOptions?.withSupply; + return !hasHeavyOptions && this.isDefaultNftFilter(filter); + } + + private isCacheableNftCount(filter: NftFilter): boolean { + return this.isDefaultNftFilter(filter); + } } diff --git a/src/endpoints/transactions/transaction.service.ts b/src/endpoints/transactions/transaction.service.ts index e2aa7523c..765b9429e 100644 --- a/src/endpoints/transactions/transaction.service.ts +++ b/src/endpoints/transactions/transaction.service.ts @@ -95,6 +95,15 @@ export class TransactionService { return this.getTransactionCountForAddress(filter.sender ?? ''); } + if (this.isCacheableTransactionCount(filter, address)) { + return await this.cachingService.getOrSet( + CacheInfo.TransactionsCount.key, + async () => await this.indexerService.getTransactionCount(filter, address), + CacheInfo.TransactionsCount.ttl, + Constants.oneSecond(), + ); + } + return await this.indexerService.getTransactionCount(filter, address); } @@ -192,6 +201,20 @@ export class TransactionService { } async getTransactions(filter: TransactionFilter, pagination: QueryPagination, queryOptions?: TransactionQueryOptions, address?: string, fields?: string[]): Promise { + if (this.isCacheableTransactionList(filter, queryOptions, fields, address)) { + const cacheInfo = CacheInfo.Transactions(pagination); + return await this.cachingService.getOrSet( + cacheInfo.key, + () => this.computeTransactions(filter, pagination, queryOptions, address, fields), + cacheInfo.ttl, + Constants.oneSecond(), + ); + } + + return await this.computeTransactions(filter, pagination, queryOptions, address, fields); + } + + private async computeTransactions(filter: TransactionFilter, pagination: QueryPagination, queryOptions?: TransactionQueryOptions, address?: string, fields?: string[]): Promise { const elasticTransactions = await this.indexerService.getTransactions(filter, pagination, address); let transactions: TransactionDetailed[] = []; @@ -820,4 +843,52 @@ export class TransactionService { return buckets; } + + private isEmptyTransactionFilter(filter: TransactionFilter): boolean { + return !filter.address && + !filter.sender && + !(filter.senders && filter.senders.length > 0) && + !(filter.receivers && filter.receivers.length > 0) && + !filter.token && + !(filter.tokens && filter.tokens.length > 0) && + !(filter.functions && filter.functions.length > 0) && + filter.senderShard === undefined && + filter.receiverShard === undefined && + !filter.miniBlockHash && + !(filter.hashes && filter.hashes.length > 0) && + filter.status === undefined && + filter.before === undefined && + filter.after === undefined && + filter.condition === undefined && + filter.order === undefined && + filter.senderOrReceiver === undefined && + filter.isScCall === undefined && + filter.isRelayed === undefined && + filter.relayer === undefined && + filter.round === undefined && + filter.withRefunds === undefined && + filter.withRelayedScresults === undefined && + filter.withTxsRelayedByAddress === undefined; + } + + private isCacheableTransactionList(filter: TransactionFilter, queryOptions?: TransactionQueryOptions, fields?: string[], address?: string): boolean { + const hasFieldSelection = Array.isArray(fields) && fields.length > 0; + if (address || hasFieldSelection || !this.isEmptyTransactionFilter(filter) || !queryOptions) { + return false; + } + + const hasAnyEnrichmentOption = queryOptions.withScResults || + queryOptions.withBlockInfo || + queryOptions.withActionTransferValue || + queryOptions.withUsername || + queryOptions.withTxsOrder || + queryOptions.withOperations !== undefined || + queryOptions.withLogs !== undefined; + + return !hasAnyEnrichmentOption; + } + + private isCacheableTransactionCount(filter: TransactionFilter, address?: string): boolean { + return !address && this.isEmptyTransactionFilter(filter); + } } diff --git a/src/test/unit/services/collections.spec.ts b/src/test/unit/services/collections.spec.ts index 30c7243fb..3d47d0e4f 100644 --- a/src/test/unit/services/collections.spec.ts +++ b/src/test/unit/services/collections.spec.ts @@ -8,11 +8,9 @@ import { PersistenceService } from "src/common/persistence/persistence.service"; import { PluginService } from "src/common/plugins/plugin.service"; import { CollectionService } from "src/endpoints/collections/collection.service"; import { CollectionFilter } from "src/endpoints/collections/entities/collection.filter"; -import { NftCollection } from "src/endpoints/collections/entities/nft.collection"; import { NftCollectionDetailed } from "src/endpoints/collections/entities/nft.collection.detailed"; import { EsdtAddressService } from "src/endpoints/esdt/esdt.address.service"; import { EsdtService } from "src/endpoints/esdt/esdt.service"; -import { NftType } from "src/endpoints/nfts/entities/nft.type"; import { CollectionRoles } from "src/endpoints/tokens/entities/collection.roles"; import { TokenAssetStatus } from "src/endpoints/tokens/entities/token.asset.status"; import { VmQueryService } from "src/endpoints/vm.query/vm.query.service"; @@ -107,7 +105,9 @@ describe('CollectionService', () => { provide: CacheService, useValue: { get: jest.fn(), - getOrSet: jest.fn(), + getOrSet: jest.fn().mockImplementation(async (_key: string, getter: () => Promise, _ttl: number) => { + return await getter(); + }), batchGetAll: jest.fn(), batchApplyAll: jest.fn(), }, @@ -133,6 +133,7 @@ describe('CollectionService', () => { { getTokenAssets: jest.fn(), getCollectionRanks: jest.fn(), + getAllTokenAssets: jest.fn().mockResolvedValue({}), }, }, { @@ -209,56 +210,17 @@ describe('CollectionService', () => { }); describe('getCollection', () => { - const propertiesToCollectionsMock: NftCollection = { - collection: 'XDAY23TEAM-f7a346', - type: NftType.NonFungibleESDT, - subType: undefined, - name: 'xPortalAchievements', - ticker: 'XDAY23TEAM', - owner: 'erd1lpc6wjh2hav6q50p8y6a44r2lhtnseqksygakjfgep6c9uduchkqphzu6t', - timestamp: 0, - canFreeze: true, - canWipe: true, - canPause: true, - canTransferNftCreateRole: true, - canChangeOwner: false, - canUpgrade: false, - canAddSpecialRoles: false, - decimals: undefined, - assets: { - website: 'https://xday.com', - description: - 'Test description.', - status: TokenAssetStatus.active, - pngUrl: 'https://media.elrond.com/tokens/asset/XDAY23TEAM-f7a346/logo.png', - name: '', - svgUrl: 'https://media.elrond.com/tokens/asset/XDAY23TEAM-f7a346/logo.svg', - extraTokens: [''], - ledgerSignature: '', - priceSource: undefined, - preferredRankAlgorithm: undefined, - lockedAccounts: undefined, - }, - scamInfo: undefined, - traits: [], - auctionStats: undefined, - isVerified: undefined, - holderCount: undefined, - nftCount: undefined, - }; - it('should return collection details for a given collection identifier', async () => { const identifier = 'XDAY23TEAM-f7a346'; jest.spyOn(indexerService, 'getCollection').mockResolvedValue(indexerCollectionMock); - jest.spyOn(service, 'applyPropertiesToCollections').mockResolvedValue([propertiesToCollectionsMock]); + jest.spyOn(service, 'getNftCollectionRolesFromGateway').mockResolvedValue([]); const result = await service.getNftCollection(identifier); expect(result).toBeInstanceOf(Object); expect(indexerService.getCollection).toHaveBeenCalledTimes(1); expect(indexerService.getCollection).toHaveBeenCalledWith(identifier); - expect(service.applyPropertiesToCollections).toHaveBeenCalledWith([identifier]); }); it('should return undefined if the collection is not found', async () => { @@ -293,23 +255,26 @@ describe('CollectionService', () => { expect(result).toBeUndefined(); }); - it('should return undefined if no additional properties are applied to the collection', async () => { - const identifier = 'XDAY23TEAM'; + it('should return collection when ES data is available', async () => { + const identifier = 'XDAY23TEAM-f7a346'; jest.spyOn(indexerService, 'getCollection').mockResolvedValue(indexerCollectionMock); - jest.spyOn(service, 'applyPropertiesToCollections').mockResolvedValue([]); + jest.spyOn(service, 'getNftCollectionRolesFromGateway').mockResolvedValue([]); const result = await service.getNftCollection(identifier); - expect(result).toBeUndefined(); + expect(result).toBeDefined(); + expect(result?.collection).toBe('XDAY23TEAM-f7a346'); }); it('should process the collection details fully', async () => { - const identifier = 'XDAY23TEAM'; + const identifier = 'XDAY23TEAM-f7a346'; jest.spyOn(indexerService, 'getCollection').mockResolvedValue(indexerCollectionMock); - jest.spyOn(service, 'applyPropertiesToCollections').mockResolvedValue([propertiesToCollectionsMock]); + jest.spyOn(service, 'getNftCollectionRolesFromGateway').mockResolvedValue([]); const result = await service.getNftCollection(identifier); expect(result).toBeInstanceOf(NftCollectionDetailed); + expect(result?.name).toBe(indexerCollectionMock.name); + expect(result?.owner).toBe(indexerCollectionMock.currentOwner); }); }); diff --git a/src/test/unit/services/transactions.spec.ts b/src/test/unit/services/transactions.spec.ts index 6a6b0c358..31f0b589c 100644 --- a/src/test/unit/services/transactions.spec.ts +++ b/src/test/unit/services/transactions.spec.ts @@ -67,7 +67,9 @@ describe('TransactionService', () => { provide: CacheService, useValue: { get: jest.fn(), - getOrSet: jest.fn(), + getOrSet: jest.fn().mockImplementation(async (_key: string, getter: () => Promise, _ttl: number) => { + return await getter(); + }), batchGetAll: jest.fn(), }, }, diff --git a/src/utils/cache.info.ts b/src/utils/cache.info.ts index 6a008e4b1..7c119fdfa 100644 --- a/src/utils/cache.info.ts +++ b/src/utils/cache.info.ts @@ -105,6 +105,18 @@ export class CacheInfo { }; } + static Transactions(queryPagination: QueryPagination): CacheInfo { + return { + key: `transactions:${queryPagination.from}:${queryPagination.size}`, + ttl: Constants.oneSecond() * 6, + }; + } + + static TransactionsCount: CacheInfo = { + key: 'transactions:count', + ttl: Constants.oneSecond() * 6, + }; + static IdentityProfilesKeybases: CacheInfo = { key: 'identityProfilesKeybases', ttl: Constants.oneHour(), @@ -174,9 +186,16 @@ export class CacheInfo { ttl: Constants.oneDay(), }; - static CollectionRanks: CacheInfo = { - key: 'collectionRanks', - ttl: Constants.oneDay(), + static Nfts(queryPagination: QueryPagination): CacheInfo { + return { + key: `nfts:${queryPagination.from}:${queryPagination.size}`, + ttl: Constants.oneSecond() * 6, + }; + } + + static NftsCount: CacheInfo = { + key: 'nfts:count', + ttl: Constants.oneSecond() * 6, }; static AccountAssets: CacheInfo = { @@ -220,6 +239,72 @@ export class CacheInfo { }; } + static CollectionTraits(identifier: string): CacheInfo { + return { + key: `collectionTraits:${identifier}`, + ttl: Constants.oneMinute() * 10, + }; + } + + static CollectionRoles(identifier: string): CacheInfo { + return { + key: `collectionRoles:${identifier}`, + ttl: Constants.oneMinute() * 5, + }; + } + + static CollectionLogo(identifier: string): CacheInfo { + return { + key: `collectionLogo:${identifier}`, + ttl: Constants.oneHour(), + }; + } + + static CollectionRanks: CacheInfo = { + key: 'collectionRanks', + ttl: Constants.oneDay(), + }; + + static CollectionRanksForIdentifier(identifier: string): CacheInfo { + return { + key: `collectionRanks:${identifier}`, + ttl: Constants.oneMinute() * 10, + }; + } + + static CollectionCountForAddress(address: string): CacheInfo { + return { + key: `collectionCount:${address}`, + ttl: Constants.oneMinute(), + }; + } + + static CollectionRolesCountForAddress(address: string): CacheInfo { + return { + key: `collectionRolesCount:${address}`, + ttl: Constants.oneMinute(), + }; + } + + static Collections(pagination: QueryPagination): CacheInfo { + return { + key: `collections:${pagination.from}:${pagination.size}`, + ttl: Constants.oneSecond() * 6, + }; + } + + static CollectionsCount: CacheInfo = { + key: 'collectionsCount', + ttl: Constants.oneSecond() * 6, + }; + + static CollectionsForAddress(address: string, pagination: QueryPagination): CacheInfo { + return { + key: `collectionsForAddress:${address}:${pagination.from}:${pagination.size}`, + ttl: Constants.oneSecond() * 6, + }; + } + static EsdtAddressesRoles(identifier: string): CacheInfo { return { key: `esdt:roles:${identifier}`,