From 36a79c501340b4b60920522f3a8903803a65c706 Mon Sep 17 00:00:00 2001 From: cfaur09 Date: Mon, 8 Dec 2025 10:26:26 +0200 Subject: [PATCH 01/10] Refactor account, NFT, and transaction services to improve concurrency and caching --- src/endpoints/accounts/account.service.ts | 54 ++++--- src/endpoints/nfts/nft.service.ts | 76 ++++++++-- .../transactions/transaction.service.ts | 139 +++++++++++++----- src/utils/cache.info.ts | 24 +++ 4 files changed, 225 insertions(+), 68 deletions(-) diff --git a/src/endpoints/accounts/account.service.ts b/src/endpoints/accounts/account.service.ts index 104a1b5be..5724cee62 100644 --- a/src/endpoints/accounts/account.service.ts +++ b/src/endpoints/accounts/account.service.ts @@ -22,6 +22,7 @@ import { GatewayService } from 'src/common/gateway/gateway.service'; import { IndexerService } from "src/common/indexer/indexer.service"; import { AccountAssets } from 'src/common/assets/entities/account.assets'; import { CacheInfo } from 'src/utils/cache.info'; +import { ConcurrencyUtils } from 'src/utils/concurrency.utils'; import { UsernameService } from '../usernames/username.service'; import { ContractUpgrades } from './entities/contract.upgrades'; import { AccountVerification } from './entities/account.verification'; @@ -356,36 +357,41 @@ export class AccountService { } } - for (const account of accounts) { - account.shard = AddressUtils.computeShard(AddressUtils.bech32Decode(account.address), shardCount); - account.assets = assets[account.address]; + await ConcurrencyUtils.executeWithConcurrencyLimit( + accounts, + async (account) => { + account.shard = AddressUtils.computeShard(AddressUtils.bech32Decode(account.address), shardCount); + account.assets = assets[account.address]; - if (options.withDeployInfo && AddressUtils.isSmartContractAddress(account.address)) { - const [deployedAt, deployTxHash] = await Promise.all([ - this.getAccountDeployedAt(account.address), - this.getAccountDeployedTxHash(account.address), - ]); + if (options.withDeployInfo && AddressUtils.isSmartContractAddress(account.address)) { + const [deployedAt, deployTxHash] = await Promise.all([ + this.getAccountDeployedAt(account.address), + this.getAccountDeployedTxHash(account.address), + ]); - account.deployedAt = deployedAt; - account.deployTxHash = deployTxHash; - } + account.deployedAt = deployedAt; + account.deployTxHash = deployTxHash; + } - if (options.withTxCount) { - account.txCount = await this.getAccountTxCount(account.address); - } + if (options.withTxCount) { + account.txCount = await this.getAccountTxCount(account.address); + } - if (options.withScrCount) { - account.scrCount = await this.getAccountScResults(account.address); - } + if (options.withScrCount) { + account.scrCount = await this.getAccountScResults(account.address); + } - if (options.withOwnerAssets && account.ownerAddress) { - account.ownerAssets = assets[account.ownerAddress]; - } + if (options.withOwnerAssets && account.ownerAddress) { + account.ownerAssets = assets[account.ownerAddress]; + } - if (verifiedAccounts && verifiedAccounts.includes(account.address)) { - account.isVerified = true; - } - } + if (verifiedAccounts && verifiedAccounts.includes(account.address)) { + account.isVerified = true; + } + }, + 6, + 'AccountService.getAccountsRaw', + ); return accounts; } diff --git a/src/endpoints/nfts/nft.service.ts b/src/endpoints/nfts/nft.service.ts index b4a5192db..7e64f5166 100644 --- a/src/endpoints/nfts/nft.service.ts +++ b/src/endpoints/nfts/nft.service.ts @@ -65,20 +65,33 @@ export class NftService { } async getNfts(queryPagination: QueryPagination, filter: NftFilter, queryOptions?: NftQueryOptions): Promise { - const { from, size } = queryPagination; + const executeGetNfts = async (): Promise => { + const { from, size } = queryPagination; - const nfts = await this.getNftsInternal({ from, size }, filter); + const nfts = await this.getNftsInternal({ from, size }, filter); - await Promise.all([ - this.conditionallyApplyAssetsAndTicker(nfts, undefined, queryOptions), - this.conditionallyApplyOwners(nfts, queryOptions), - this.conditionallyApplySupply(nfts, queryOptions), - this.batchProcessNfts(nfts), - ]); + await Promise.all([ + this.conditionallyApplyAssetsAndTicker(nfts, undefined, queryOptions), + this.conditionallyApplyOwners(nfts, queryOptions), + this.conditionallyApplySupply(nfts, queryOptions), + this.batchProcessNfts(nfts), + ]); - await this.batchApplyUnlockFields(nfts); + await this.batchApplyUnlockFields(nfts); - return nfts; + return nfts; + }; + + if (this.isCacheableNftList(filter, queryOptions)) { + const cacheInfo = CacheInfo.Nfts(queryPagination); + return await this.cachingService.getOrSet( + cacheInfo.key, + executeGetNfts, + cacheInfo.ttl, + ); + } + + return await executeGetNfts(); } private async batchProcessNfts(nfts: Nft[], fields?: string[]) { @@ -501,6 +514,14 @@ export class NftService { } async getNftCount(filter: NftFilter): Promise { + if (this.isCacheableNftCount(filter)) { + return await this.cachingService.getOrSet( + CacheInfo.NftsCount.key, + async () => await this.indexerService.getNftCount(filter), + CacheInfo.NftsCount.ttl, + ); + } + return await this.indexerService.getNftCount(filter); } @@ -730,4 +751,39 @@ export class NftService { this.logger.error(error); } } + + private isDefaultNftFilter(filter: NftFilter): boolean { + return !filter.search && + !(filter.identifiers && filter.identifiers.length > 0) && + !(filter.type && filter.type.length > 0) && + !(filter.subType && filter.subType.length > 0) && + !filter.collection && + !(filter.collections && filter.collections.length > 0) && + !(filter.tags && filter.tags.length > 0) && + !filter.name && + !filter.creator && + filter.hasUris === undefined && + filter.includeFlagged === undefined && + filter.before === undefined && + filter.after === undefined && + filter.nonceBefore === undefined && + filter.nonceAfter === undefined && + filter.isWhitelistedStorage === undefined && + filter.isNsfw === undefined && + filter.isScam === undefined && + filter.scamType === undefined && + !filter.traits && + filter.excludeMetaESDT === undefined && + filter.sort === undefined && + filter.order === undefined; + } + + private isCacheableNftList(filter: NftFilter, queryOptions?: NftQueryOptions): boolean { + const hasHeavyOptions = queryOptions?.withOwner || queryOptions?.withSupply; + return !hasHeavyOptions && this.isDefaultNftFilter(filter); + } + + private isCacheableNftCount(filter: NftFilter): boolean { + return this.isDefaultNftFilter(filter); + } } diff --git a/src/endpoints/transactions/transaction.service.ts b/src/endpoints/transactions/transaction.service.ts index e2aa7523c..be7220d02 100644 --- a/src/endpoints/transactions/transaction.service.ts +++ b/src/endpoints/transactions/transaction.service.ts @@ -95,6 +95,15 @@ export class TransactionService { return this.getTransactionCountForAddress(filter.sender ?? ''); } + if (this.isCacheableTransactionCount(filter, address)) { + return await this.cachingService.getOrSet( + CacheInfo.TransactionsCount.key, + async () => await this.indexerService.getTransactionCount(filter, address), + CacheInfo.TransactionsCount.ttl, + Constants.oneSecond(), + ); + } + return await this.indexerService.getTransactionCount(filter, address); } @@ -192,53 +201,67 @@ export class TransactionService { } async getTransactions(filter: TransactionFilter, pagination: QueryPagination, queryOptions?: TransactionQueryOptions, address?: string, fields?: string[]): Promise { - const elasticTransactions = await this.indexerService.getTransactions(filter, pagination, address); + const computeTransactions = async (): Promise => { + const elasticTransactions = await this.indexerService.getTransactions(filter, pagination, address); - let transactions: TransactionDetailed[] = []; - transactions = elasticTransactions.map(x => ApiUtils.mergeObjects(new TransactionDetailed(), x)); + let transactions: TransactionDetailed[] = []; + transactions = elasticTransactions.map(x => ApiUtils.mergeObjects(new TransactionDetailed(), x)); - const hasSenderFilter = filter.sender || (filter.senders && filter.senders.length > 0); - const hasReceiverFilter = filter.receivers && filter.receivers.length > 0; + const hasSenderFilter = filter.sender || (filter.senders && filter.senders.length > 0); + const hasReceiverFilter = filter.receivers && filter.receivers.length > 0; - if (address && !hasSenderFilter && !hasReceiverFilter) { - transactions = this.reorderAccountSentTransactionsByNonce(transactions, address); - } + if (address && !hasSenderFilter && !hasReceiverFilter) { + transactions = this.reorderAccountSentTransactionsByNonce(transactions, address); + } - if (filter.hashes) { - const txHashes: string[] = filter.hashes; - const elasticHashes = elasticTransactions.map(({ txHash }: any) => txHash); - const missingHashes: string[] = txHashes.except(elasticHashes); + if (filter.hashes) { + const txHashes: string[] = filter.hashes; + const elasticHashes = elasticTransactions.map(({ txHash }: any) => txHash); + const missingHashes: string[] = txHashes.except(elasticHashes); - const gatewayTransactions = await Promise.all(missingHashes.map((txHash) => this.transactionGetService.tryGetTransactionFromGatewayForList(txHash))); - for (const gatewayTransaction of gatewayTransactions) { - if (gatewayTransaction) { - transactions.push(ApiUtils.mergeObjects(new TransactionDetailed(), gatewayTransaction)); + const gatewayTransactions = await Promise.all(missingHashes.map((txHash) => this.transactionGetService.tryGetTransactionFromGatewayForList(txHash))); + for (const gatewayTransaction of gatewayTransactions) { + if (gatewayTransaction) { + transactions.push(ApiUtils.mergeObjects(new TransactionDetailed(), gatewayTransaction)); + } } } - } - if ((queryOptions && queryOptions.withBlockInfo) || (fields && fields.includesSome(['senderBlockHash', 'receiverBlockHash', 'senderBlockNonce', 'receiverBlockNonce']))) { - await this.applyBlockInfo(transactions); - } + if ((queryOptions && queryOptions.withBlockInfo) || (fields && fields.includesSome(['senderBlockHash', 'receiverBlockHash', 'senderBlockNonce', 'receiverBlockNonce']))) { + await this.applyBlockInfo(transactions); + } - if (queryOptions && (queryOptions.withScResults || queryOptions.withOperations || queryOptions.withLogs)) { - queryOptions.withScResultLogs = queryOptions.withLogs; - transactions = await this.getExtraDetailsForTransactions(elasticTransactions, transactions, queryOptions); - } + if (queryOptions && (queryOptions.withScResults || queryOptions.withOperations || queryOptions.withLogs)) { + queryOptions.withScResultLogs = queryOptions.withLogs; + transactions = await this.getExtraDetailsForTransactions(elasticTransactions, transactions, queryOptions); + } - for (const transaction of transactions) { - transaction.type = undefined; - } + for (const transaction of transactions) { + transaction.type = undefined; + } - await this.processTransactions(transactions, { - withScamInfo: queryOptions?.withScamInfo ?? false, - withUsername: queryOptions?.withUsername ?? false, - withActionTransferValue: queryOptions?.withActionTransferValue ?? false, - }); + await this.processTransactions(transactions, { + withScamInfo: queryOptions?.withScamInfo ?? false, + withUsername: queryOptions?.withUsername ?? false, + withActionTransferValue: queryOptions?.withActionTransferValue ?? false, + }); + + this.processRelayedInfo(transactions); + + return transactions; + }; - this.processRelayedInfo(transactions); + if (this.isCacheableTransactionList(filter, queryOptions, fields, address)) { + const cacheInfo = CacheInfo.Transactions(pagination); + return await this.cachingService.getOrSet( + cacheInfo.key, + computeTransactions, + cacheInfo.ttl, + Constants.oneSecond(), + ); + } - return transactions; + return await computeTransactions(); } private getAssetsFromUsername(username: string | null | undefined): AccountAssets | undefined { @@ -820,4 +843,52 @@ export class TransactionService { return buckets; } + + private isEmptyTransactionFilter(filter: TransactionFilter): boolean { + return !filter.address && + !filter.sender && + !(filter.senders && filter.senders.length > 0) && + !(filter.receivers && filter.receivers.length > 0) && + !filter.token && + !(filter.tokens && filter.tokens.length > 0) && + !(filter.functions && filter.functions.length > 0) && + filter.senderShard === undefined && + filter.receiverShard === undefined && + !filter.miniBlockHash && + !(filter.hashes && filter.hashes.length > 0) && + filter.status === undefined && + filter.before === undefined && + filter.after === undefined && + filter.condition === undefined && + filter.order === undefined && + filter.senderOrReceiver === undefined && + filter.isScCall === undefined && + filter.isRelayed === undefined && + filter.relayer === undefined && + filter.round === undefined && + filter.withRefunds === undefined && + filter.withRelayedScresults === undefined && + filter.withTxsRelayedByAddress === undefined; + } + + private isCacheableTransactionList(filter: TransactionFilter, queryOptions?: TransactionQueryOptions, fields?: string[], address?: string): boolean { + const hasFieldSelection = Array.isArray(fields) && fields.length > 0; + if (address || hasFieldSelection || !this.isEmptyTransactionFilter(filter) || !queryOptions) { + return false; + } + + const hasNonDefaultOptions = queryOptions.withScResults || + queryOptions.withBlockInfo || + queryOptions.withActionTransferValue || + queryOptions.withUsername || + queryOptions.withTxsOrder || + queryOptions.withOperations === false || + queryOptions.withLogs === false; + + return !hasNonDefaultOptions; + } + + private isCacheableTransactionCount(filter: TransactionFilter, address?: string): boolean { + return !address && this.isEmptyTransactionFilter(filter); + } } diff --git a/src/utils/cache.info.ts b/src/utils/cache.info.ts index 00059ae49..d2bd54b6d 100644 --- a/src/utils/cache.info.ts +++ b/src/utils/cache.info.ts @@ -105,6 +105,18 @@ export class CacheInfo { }; } + static Transactions(queryPagination: QueryPagination): CacheInfo { + return { + key: `transactions:${queryPagination.from}:${queryPagination.size}`, + ttl: Constants.oneSecond() * 6, + }; + } + + static TransactionsCount: CacheInfo = { + key: 'transactions:count', + ttl: Constants.oneSecond() * 6, + }; + static IdentityProfilesKeybases: CacheInfo = { key: 'identityProfilesKeybases', ttl: Constants.oneHour(), @@ -174,6 +186,18 @@ export class CacheInfo { ttl: Constants.oneDay(), }; + static Nfts(queryPagination: QueryPagination): CacheInfo { + return { + key: `nfts:${queryPagination.from}:${queryPagination.size}`, + ttl: Constants.oneSecond() * 6, + }; + } + + static NftsCount: CacheInfo = { + key: 'nfts:count', + ttl: Constants.oneSecond() * 6, + }; + static CollectionRanks: CacheInfo = { key: 'collectionRanks', ttl: Constants.oneDay(), From c8c7235dbf788abe729379c0330dd32ad5de093f Mon Sep 17 00:00:00 2001 From: cfaur09 Date: Mon, 8 Dec 2025 11:23:47 +0200 Subject: [PATCH 02/10] improve NFT bulk fetch + applyOwner --- src/endpoints/nfts/nft.service.ts | 57 +++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 18 deletions(-) diff --git a/src/endpoints/nfts/nft.service.ts b/src/endpoints/nfts/nft.service.ts index 7e64f5166..30ee180eb 100644 --- a/src/endpoints/nfts/nft.service.ts +++ b/src/endpoints/nfts/nft.service.ts @@ -31,6 +31,7 @@ import { SortCollectionNfts } from "../collections/entities/sort.collection.nfts import { TokenAssets } from "src/common/assets/entities/token.assets"; import { ScamInfo } from "src/common/entities/scam-info.dto"; import { NftSubType } from "./entities/nft.sub.type"; +import { ConcurrencyUtils } from "src/utils/concurrency.utils"; @Injectable() export class NftService { @@ -127,21 +128,15 @@ export class NftService { return; } - const nftsIdentifiers = nfts.filter(x => x.type === NftType.NonFungibleESDT).map(x => x.identifier); + const nftsIdentifiers = nfts + .filter(x => x.type === NftType.NonFungibleESDT) + .map(x => x.identifier); if (nftsIdentifiers.length === 0) { return; } - const accountsEsdts = await this.getAccountEsdtByIdentifiers(nftsIdentifiers, { - from: 0, - size: nftsIdentifiers.length, - }); - - const ownerMap = accountsEsdts.reduce((acc: Record, accountEsdt: any) => { - acc[accountEsdt.identifier] = accountEsdt.address; - return acc; - }, {}); + const ownerMap = await this.getOwnersBulk(nftsIdentifiers); for (const nft of nfts) { if (nft.type === NftType.NonFungibleESDT && ownerMap[nft.identifier]) { @@ -195,6 +190,29 @@ export class NftService { ); } + private async getOwnersBulk(identifiers: string[], chunkSize: number = 512, concurrencyLimit: number = 4): Promise> { + if (identifiers.length === 0) { + return {}; + } + + const chunks = BatchUtils.splitArrayIntoChunks(identifiers.distinct(), chunkSize); + const results = await ConcurrencyUtils.executeWithConcurrencyLimit( + chunks, + async (chunk) => await this.getAccountEsdtByIdentifiers(chunk, { from: 0, size: chunk.length }), + concurrencyLimit, + 'NftService.getOwnersBulk' + ); + + const ownerMap: Record = {}; + for (const chunkResult of results) { + for (const accountEsdt of chunkResult ?? []) { + ownerMap[accountEsdt.identifier] = accountEsdt.address; + } + } + + return ownerMap; + } + private async batchApplyMedia(nfts: Nft[], fields?: string[]) { if (fields && !fields.includes('media')) { return; @@ -570,15 +588,18 @@ export class NftService { } private async getNftsInternalByIdentifiers(identifiers: string[]): Promise { - const chunks = BatchUtils.splitArrayIntoChunks(identifiers, 1024); - const result: Nft[] = []; - for (const identifiers of chunks) { - const internalNfts = await this.getNftsInternal(new QueryPagination({ from: 0, size: identifiers.length }), new NftFilter({ identifiers })); - - result.push(...internalNfts); - } + const chunks = BatchUtils.splitArrayIntoChunks(identifiers, 512); + const results = await ConcurrencyUtils.executeWithConcurrencyLimit( + chunks, + async (chunk) => await this.getNftsInternal( + new QueryPagination({ from: 0, size: chunk.length }), + new NftFilter({ identifiers: chunk }) + ), + 4, + 'NftService.getNftsInternalByIdentifiers' + ); - return result; + return results.flat(); } private async applyPriceUsd(nft: NftAccount, fields?: string[]) { From cc700f4972803886ba603a75ecfa67fc5e502480 Mon Sep 17 00:00:00 2001 From: cfaur09 Date: Mon, 8 Dec 2025 22:03:58 +0200 Subject: [PATCH 03/10] implement ConcurrencyUtils in collection service --- .../collections/collection.service.ts | 89 +++++++++++++++---- src/utils/cache.info.ts | 21 +++++ 2 files changed, 95 insertions(+), 15 deletions(-) diff --git a/src/endpoints/collections/collection.service.ts b/src/endpoints/collections/collection.service.ts index ce8378053..ca0b1ce68 100644 --- a/src/endpoints/collections/collection.service.ts +++ b/src/endpoints/collections/collection.service.ts @@ -29,6 +29,7 @@ import { CollectionLogo } from "./entities/collection.logo"; import { ScamInfo } from "src/common/entities/scam-info.dto"; import { NftType as ElasticNftType } from "src/common/indexer/entities/nft.type"; import { NftSubType } from "../nfts/entities/nft.sub.type"; +import { ConcurrencyUtils } from "src/utils/concurrency.utils"; @Injectable() export class CollectionService { @@ -159,24 +160,46 @@ export class CollectionService { } async batchGetCollectionsProperties(identifiers: string[]): Promise<{ [key: string]: TokenProperties | undefined }> { - if (this.apiConfigService.getCollectionPropertiesFromGateway()) { - return await this.getCollectionProperties(identifiers); + const result: { [key: string]: TokenProperties | undefined } = {}; + const chunks = this.splitIntoChunks(identifiers, 300); + + const chunkResults = await ConcurrencyUtils.executeWithConcurrencyLimit( + chunks, + async (chunk) => { + if (this.apiConfigService.getCollectionPropertiesFromGateway()) { + return await this.getCollectionProperties(chunk); + } + return await this.getEsdtProperties(chunk); + }, + 4, + 'CollectionService.batchGetCollectionsProperties' + ); + + for (const chunkResult of chunkResults) { + Object.assign(result, chunkResult); } - return await this.getEsdtProperties(identifiers); + return result; } async batchGetCollectionsAssets(identifiers: string[]): Promise<{ [key: string]: TokenAssets | undefined }> { const collectionsAssets: { [key: string]: TokenAssets | undefined } = {}; - const allAssets = await this.assetsService.getAllTokenAssets(); - - await this.cachingService.batchApplyAll( - identifiers, - identifier => CacheInfo.EsdtAssets(identifier).key, - identifier => Promise.resolve(allAssets[identifier]), - (identifier, properties) => collectionsAssets[identifier] = properties, - CacheInfo.EsdtAssets('').ttl + const chunks = this.splitIntoChunks(identifiers, 300); + + await ConcurrencyUtils.executeWithConcurrencyLimit( + chunks, + async (chunk) => { + await this.cachingService.batchApplyAll( + chunk, + identifier => CacheInfo.EsdtAssets(identifier).key, + identifier => Promise.resolve(allAssets[identifier]), + (identifier, properties) => collectionsAssets[identifier] = properties, + CacheInfo.EsdtAssets('').ttl + ); + }, + 4, + 'CollectionService.batchGetCollectionsAssets' ); return collectionsAssets; @@ -230,7 +253,7 @@ export class CollectionService { this.applyPropertiesToCollectionFromElasticSearch(collectionDetailed, elasticCollection); - collectionDetailed.traits = await this.persistenceService.getCollectionTraits(identifier) ?? []; + collectionDetailed.traits = await this.getCollectionTraitsCached(identifier); await this.applyCollectionRoles(collectionDetailed, elasticCollection); @@ -238,7 +261,7 @@ export class CollectionService { } async applyCollectionRoles(collection: NftCollectionDetailed | TokenDetailed, elasticCollection: any) { - collection.roles = await this.getNftCollectionRolesFromGateway(elasticCollection); + collection.roles = await this.getCollectionRolesCached(elasticCollection.token, elasticCollection); const isTransferProhibitedByDefault = collection.roles?.some(x => x.canTransfer === true) === true; collection.canTransfer = !isTransferProhibitedByDefault; if (collection.canTransfer) { @@ -383,7 +406,7 @@ export class CollectionService { } async getLogoPng(identifier: string): Promise { - const collectionLogo = await this.getCollectionLogo(identifier); + const collectionLogo = await this.getCollectionLogoCached(identifier); if (!collectionLogo) { return; } @@ -392,7 +415,7 @@ export class CollectionService { } async getLogoSvg(identifier: string): Promise { - const collectionLogo = await this.getCollectionLogo(identifier); + const collectionLogo = await this.getCollectionLogoCached(identifier); if (!collectionLogo) { return; } @@ -427,4 +450,40 @@ export class CollectionService { return collectionsProperties; } + + private splitIntoChunks(items: T[], chunkSize: number): T[][] { + if (chunkSize <= 0) { + return [items]; + } + + const chunks: T[][] = []; + for (let i = 0; i < items.length; i += chunkSize) { + chunks.push(items.slice(i, i + chunkSize)); + } + return chunks; + } + + private async getCollectionTraitsCached(identifier: string): Promise { + return await this.cachingService.getOrSet( + CacheInfo.CollectionTraits(identifier).key, + async () => await this.persistenceService.getCollectionTraits(identifier) ?? [], + CacheInfo.CollectionTraits(identifier).ttl, + ); + } + + private async getCollectionRolesCached(identifier: string, elasticCollection: any): Promise { + return await this.cachingService.getOrSet( + CacheInfo.CollectionRoles(identifier).key, + async () => await this.getNftCollectionRolesFromGateway(elasticCollection), + CacheInfo.CollectionRoles(identifier).ttl, + ); + } + + private async getCollectionLogoCached(identifier: string): Promise { + return await this.cachingService.getOrSet( + CacheInfo.CollectionLogo(identifier).key, + async () => await this.getCollectionLogo(identifier), + CacheInfo.CollectionLogo(identifier).ttl, + ); + } } diff --git a/src/utils/cache.info.ts b/src/utils/cache.info.ts index d2bd54b6d..37610717b 100644 --- a/src/utils/cache.info.ts +++ b/src/utils/cache.info.ts @@ -244,6 +244,27 @@ export class CacheInfo { }; } + static CollectionTraits(identifier: string): CacheInfo { + return { + key: `collectionTraits:${identifier}`, + ttl: Constants.oneMinute() * 10, + }; + } + + static CollectionRoles(identifier: string): CacheInfo { + return { + key: `collectionRoles:${identifier}`, + ttl: Constants.oneMinute() * 5, + }; + } + + static CollectionLogo(identifier: string): CacheInfo { + return { + key: `collectionLogo:${identifier}`, + ttl: Constants.oneHour(), + }; + } + static EsdtAddressesRoles(identifier: string): CacheInfo { return { key: `esdt:roles:${identifier}`, From 999b183f21662009aa0fabd808d02886b1ae131d Mon Sep 17 00:00:00 2001 From: cfaur09 Date: Thu, 11 Dec 2025 17:23:10 +0200 Subject: [PATCH 04/10] collection service improvments --- .../collections/collection.service.ts | 96 +++++++++++++++---- src/utils/cache.info.ts | 43 ++++++++- 2 files changed, 118 insertions(+), 21 deletions(-) diff --git a/src/endpoints/collections/collection.service.ts b/src/endpoints/collections/collection.service.ts index ca0b1ce68..3b946ca4b 100644 --- a/src/endpoints/collections/collection.service.ts +++ b/src/endpoints/collections/collection.service.ts @@ -51,8 +51,40 @@ export class CollectionService { } async getNftCollections(pagination: QueryPagination, filter: CollectionFilter): Promise { - const tokenCollections = await this.indexerService.getNftCollections(pagination, filter); - return await this.processNftCollections(tokenCollections); + const executeGetCollections = async (): Promise => { + const tokenCollections = await this.indexerService.getNftCollections(pagination, filter); + return await this.processNftCollections(tokenCollections); + }; + + if (this.isCacheableCollectionFilter(filter)) { + return await this.cachingService.getOrSet( + CacheInfo.Collections(pagination).key, + executeGetCollections, + CacheInfo.Collections(pagination).ttl, + ); + } + + return await executeGetCollections(); + } + + private isCacheableCollectionFilter(filter: CollectionFilter): boolean { + return !filter.collection && + !(filter.identifiers && filter.identifiers.length > 0) && + !(filter.type && filter.type.length > 0) && + !(filter.subType && filter.subType.length > 0) && + !filter.search && + !filter.owner && + filter.before === undefined && + filter.after === undefined && + filter.canCreate === undefined && + filter.canBurn === undefined && + filter.canAddQuantity === undefined && + filter.canUpdateAttributes === undefined && + filter.canAddUri === undefined && + filter.canTransferRole === undefined && + filter.excludeMetaESDT === undefined && + filter.sort === undefined && + filter.order === undefined; } async getNftCollectionsByIds(identifiers: Array): Promise { @@ -206,6 +238,14 @@ export class CollectionService { } async getNftCollectionCount(filter: CollectionFilter): Promise { + if (this.isCacheableCollectionFilter(filter)) { + return await this.cachingService.getOrSet( + CacheInfo.CollectionsCount.key, + async () => await this.indexerService.getNftCollectionCount(filter), + CacheInfo.CollectionsCount.ttl, + ); + } + return await this.indexerService.getNftCollectionCount(filter); } @@ -224,7 +264,11 @@ export class CollectionService { return undefined; } - return await this.assetsService.getCollectionRanks(identifier); + return await this.cachingService.getOrSet( + CacheInfo.CollectionRanksForIdentifier(identifier).key, + async () => await this.assetsService.getCollectionRanks(identifier), + CacheInfo.CollectionRanksForIdentifier(identifier).ttl, + ); } async getNftCollection(identifier: string): Promise { @@ -353,9 +397,15 @@ export class CollectionService { } async getCollectionCountForAddress(address: string, filter: CollectionFilter): Promise { - const collections = await this.getCollectionsForAddress(address, filter, new QueryPagination({ from: 0, size: 10000 })); - - return collections.length; + const filterKey = JSON.stringify(filter ?? {}); + return await this.cachingService.getOrSet( + CacheInfo.CollectionCountForAddress(address, filterKey).key, + async () => { + const collections = await this.getCollectionsForAddress(address, filter, new QueryPagination({ from: 0, size: 10000 })); + return collections.length; + }, + CacheInfo.CollectionCountForAddress(address, filterKey).ttl, + ); } async getCollectionForAddress(address: string, identifier: string): Promise { @@ -376,16 +426,25 @@ export class CollectionService { async getCollectionsForAddress(address: string, filter: CollectionFilter, pagination: QueryPagination): Promise { const collectionsRaw = await this.indexerService.getCollectionsForAddress(address, filter, pagination); - const collections = await this.getNftCollections( - new QueryPagination({ from: 0, size: collectionsRaw.length }), - new CollectionFilter({ identifiers: collectionsRaw.map((x: any) => x.collection) }) - ); - const accountCollections = collections.map(collection => ApiUtils.mergeObjects(new NftCollectionAccount(), collection)); + if (collectionsRaw.length === 0) { + return []; + } - for (const collection of accountCollections) { - const item = collectionsRaw.find(x => x.collection === collection.collection); - if (item) { - collection.count = item.count; + const identifiers = collectionsRaw.map((x: any) => x.collection); + const collections = await this.getNftCollectionsByIds(identifiers); + + const collectionMap = new Map(); + for (const collection of collections) { + collectionMap.set(collection.collection, collection); + } + + const accountCollections: NftCollectionAccount[] = []; + for (const raw of collectionsRaw) { + const collection = collectionMap.get(raw.collection); + if (collection) { + const accountCollection = ApiUtils.mergeObjects(new NftCollectionAccount(), collection); + accountCollection.count = raw.count; + accountCollections.push(accountCollection); } } @@ -393,7 +452,12 @@ export class CollectionService { } async getCollectionCountForAddressWithRoles(address: string, filter: CollectionFilter): Promise { - return await this.esdtAddressService.getCollectionCountForAddressFromElastic(address, filter); + const filterKey = JSON.stringify(filter ?? {}); + return await this.cachingService.getOrSet( + CacheInfo.CollectionRolesCountForAddress(address, filterKey).key, + async () => await this.esdtAddressService.getCollectionCountForAddressFromElastic(address, filter), + CacheInfo.CollectionRolesCountForAddress(address, filterKey).ttl, + ); } private async getCollectionLogo(identifier: string): Promise { diff --git a/src/utils/cache.info.ts b/src/utils/cache.info.ts index d9bae11c0..310661ada 100644 --- a/src/utils/cache.info.ts +++ b/src/utils/cache.info.ts @@ -198,11 +198,6 @@ export class CacheInfo { ttl: Constants.oneSecond() * 6, }; - static CollectionRanks: CacheInfo = { - key: 'collectionRanks', - ttl: Constants.oneDay(), - }; - static AccountAssets: CacheInfo = { key: 'accountLabels', ttl: Constants.oneDay(), @@ -265,6 +260,44 @@ export class CacheInfo { }; } + static CollectionRanks: CacheInfo = { + key: 'collectionRanks', + ttl: Constants.oneDay(), + }; + + static CollectionRanksForIdentifier(identifier: string): CacheInfo { + return { + key: `collectionRanks:${identifier}`, + ttl: Constants.oneMinute() * 10, + }; + } + + static CollectionCountForAddress(address: string, filterKey: string): CacheInfo { + return { + key: `collectionCount:${address}:${filterKey}`, + ttl: Constants.oneMinute(), + }; + } + + static CollectionRolesCountForAddress(address: string, filterKey: string): CacheInfo { + return { + key: `collectionRolesCount:${address}:${filterKey}`, + ttl: Constants.oneMinute(), + }; + } + + static Collections(pagination: QueryPagination): CacheInfo { + return { + key: `collections:${pagination.from}:${pagination.size}`, + ttl: Constants.oneSecond() * 6, + }; + } + + static CollectionsCount: CacheInfo = { + key: 'collectionsCount', + ttl: Constants.oneSecond() * 6, + }; + static EsdtAddressesRoles(identifier: string): CacheInfo { return { key: `esdt:roles:${identifier}`, From 0b417360a8d72ad994c634289dc560a0dc2520f1 Mon Sep 17 00:00:00 2001 From: cfaur09 Date: Thu, 11 Dec 2025 17:35:27 +0200 Subject: [PATCH 05/10] fix spec --- src/test/unit/services/collections.spec.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/test/unit/services/collections.spec.ts b/src/test/unit/services/collections.spec.ts index 30c7243fb..163a5441c 100644 --- a/src/test/unit/services/collections.spec.ts +++ b/src/test/unit/services/collections.spec.ts @@ -107,7 +107,9 @@ describe('CollectionService', () => { provide: CacheService, useValue: { get: jest.fn(), - getOrSet: jest.fn(), + getOrSet: jest.fn().mockImplementation(async (_key: string, getter: () => Promise, _ttl: number) => { + return await getter(); + }), batchGetAll: jest.fn(), batchApplyAll: jest.fn(), }, @@ -252,6 +254,7 @@ describe('CollectionService', () => { jest.spyOn(indexerService, 'getCollection').mockResolvedValue(indexerCollectionMock); jest.spyOn(service, 'applyPropertiesToCollections').mockResolvedValue([propertiesToCollectionsMock]); + jest.spyOn(service, 'getNftCollectionRolesFromGateway').mockResolvedValue([]); const result = await service.getNftCollection(identifier); @@ -304,9 +307,10 @@ describe('CollectionService', () => { }); it('should process the collection details fully', async () => { - const identifier = 'XDAY23TEAM'; + const identifier = 'XDAY23TEAM-f7a346'; jest.spyOn(indexerService, 'getCollection').mockResolvedValue(indexerCollectionMock); jest.spyOn(service, 'applyPropertiesToCollections').mockResolvedValue([propertiesToCollectionsMock]); + jest.spyOn(service, 'getNftCollectionRolesFromGateway').mockResolvedValue([]); const result = await service.getNftCollection(identifier); expect(result).toBeInstanceOf(NftCollectionDetailed); From 9e220abe9a928b861bececc2b145dad7fe0545ef Mon Sep 17 00:00:00 2001 From: cfaur09 Date: Thu, 11 Dec 2025 17:44:44 +0200 Subject: [PATCH 06/10] fixes --- .../collections/collection.service.ts | 97 ++++++++++++------- src/endpoints/nfts/nft.service.ts | 9 +- src/utils/cache.info.ts | 15 ++- 3 files changed, 79 insertions(+), 42 deletions(-) diff --git a/src/endpoints/collections/collection.service.ts b/src/endpoints/collections/collection.service.ts index 3b946ca4b..369831f62 100644 --- a/src/endpoints/collections/collection.service.ts +++ b/src/endpoints/collections/collection.service.ts @@ -397,15 +397,20 @@ export class CollectionService { } async getCollectionCountForAddress(address: string, filter: CollectionFilter): Promise { - const filterKey = JSON.stringify(filter ?? {}); - return await this.cachingService.getOrSet( - CacheInfo.CollectionCountForAddress(address, filterKey).key, - async () => { - const collections = await this.getCollectionsForAddress(address, filter, new QueryPagination({ from: 0, size: 10000 })); - return collections.length; - }, - CacheInfo.CollectionCountForAddress(address, filterKey).ttl, - ); + const executeCount = async () => { + const collections = await this.getCollectionsForAddress(address, filter, new QueryPagination({ from: 0, size: 10000 })); + return collections.length; + }; + + if (this.isDefaultAddressCollectionFilter(filter)) { + return await this.cachingService.getOrSet( + CacheInfo.CollectionCountForAddress(address).key, + executeCount, + CacheInfo.CollectionCountForAddress(address).ttl, + ); + } + + return await executeCount(); } async getCollectionForAddress(address: string, identifier: string): Promise { @@ -424,40 +429,66 @@ export class CollectionService { } async getCollectionsForAddress(address: string, filter: CollectionFilter, pagination: QueryPagination): Promise { - const collectionsRaw = await this.indexerService.getCollectionsForAddress(address, filter, pagination); + const executeGetCollections = async (): Promise => { + const collectionsRaw = await this.indexerService.getCollectionsForAddress(address, filter, pagination); - if (collectionsRaw.length === 0) { - return []; - } + if (collectionsRaw.length === 0) { + return []; + } - const identifiers = collectionsRaw.map((x: any) => x.collection); - const collections = await this.getNftCollectionsByIds(identifiers); + const identifiers = collectionsRaw.map((x: any) => x.collection); + const collections = await this.getNftCollectionsByIds(identifiers); - const collectionMap = new Map(); - for (const collection of collections) { - collectionMap.set(collection.collection, collection); - } + const collectionMap = new Map(); + for (const collection of collections) { + collectionMap.set(collection.collection, collection); + } - const accountCollections: NftCollectionAccount[] = []; - for (const raw of collectionsRaw) { - const collection = collectionMap.get(raw.collection); - if (collection) { - const accountCollection = ApiUtils.mergeObjects(new NftCollectionAccount(), collection); - accountCollection.count = raw.count; - accountCollections.push(accountCollection); + const accountCollections: NftCollectionAccount[] = []; + for (const raw of collectionsRaw) { + const collection = collectionMap.get(raw.collection); + if (collection) { + const accountCollection = ApiUtils.mergeObjects(new NftCollectionAccount(), collection); + accountCollection.count = raw.count; + accountCollections.push(accountCollection); + } } + + return accountCollections; + }; + + if (this.isDefaultAddressCollectionFilter(filter)) { + const cacheInfo = CacheInfo.CollectionsForAddress(address, pagination); + return await this.cachingService.getOrSet( + cacheInfo.key, + executeGetCollections, + cacheInfo.ttl, + ); } - return accountCollections; + return await executeGetCollections(); + } + + private isDefaultAddressCollectionFilter(filter: CollectionFilter): boolean { + return !filter.search && + !(filter.type && filter.type.length > 0) && + !(filter.subType && filter.subType.length > 0) && + !filter.excludeMetaESDT && + !filter.collection; } async getCollectionCountForAddressWithRoles(address: string, filter: CollectionFilter): Promise { - const filterKey = JSON.stringify(filter ?? {}); - return await this.cachingService.getOrSet( - CacheInfo.CollectionRolesCountForAddress(address, filterKey).key, - async () => await this.esdtAddressService.getCollectionCountForAddressFromElastic(address, filter), - CacheInfo.CollectionRolesCountForAddress(address, filterKey).ttl, - ); + const executeCount = async () => await this.esdtAddressService.getCollectionCountForAddressFromElastic(address, filter); + + if (this.isDefaultAddressCollectionFilter(filter)) { + return await this.cachingService.getOrSet( + CacheInfo.CollectionRolesCountForAddress(address).key, + executeCount, + CacheInfo.CollectionRolesCountForAddress(address).ttl, + ); + } + + return await executeCount(); } private async getCollectionLogo(identifier: string): Promise { diff --git a/src/endpoints/nfts/nft.service.ts b/src/endpoints/nfts/nft.service.ts index 30ee180eb..d81bb7268 100644 --- a/src/endpoints/nfts/nft.service.ts +++ b/src/endpoints/nfts/nft.service.ts @@ -545,10 +545,11 @@ export class NftService { async getNftsForAddress(address: string, queryPagination: QueryPagination, filter: NftFilter, fields?: string[], queryOptions?: NftQueryOptions, source?: EsdtDataSource): Promise { let nfts = await this.esdtAddressService.getNftsForAddress(address, filter, queryPagination, source, queryOptions); - for (const nft of nfts) { + + await Promise.all(nfts.map(async (nft) => { await this.applyAssetsAndTicker(nft, fields); await this.applyPriceUsd(nft, fields); - } + })); if (queryOptions && queryOptions.withSupply) { const supplyNfts = nfts.filter(nft => nft.type.in(NftType.SemiFungibleESDT, NftType.MetaESDT)); @@ -580,9 +581,7 @@ export class NftService { nfts = this.applyScamFilter(nfts, filter); - for (const nft of nfts) { - await this.applyUnlockFields(nft, fields); - } + await Promise.all(nfts.map(nft => this.applyUnlockFields(nft, fields))); return nfts; } diff --git a/src/utils/cache.info.ts b/src/utils/cache.info.ts index 310661ada..7c119fdfa 100644 --- a/src/utils/cache.info.ts +++ b/src/utils/cache.info.ts @@ -272,16 +272,16 @@ export class CacheInfo { }; } - static CollectionCountForAddress(address: string, filterKey: string): CacheInfo { + static CollectionCountForAddress(address: string): CacheInfo { return { - key: `collectionCount:${address}:${filterKey}`, + key: `collectionCount:${address}`, ttl: Constants.oneMinute(), }; } - static CollectionRolesCountForAddress(address: string, filterKey: string): CacheInfo { + static CollectionRolesCountForAddress(address: string): CacheInfo { return { - key: `collectionRolesCount:${address}:${filterKey}`, + key: `collectionRolesCount:${address}`, ttl: Constants.oneMinute(), }; } @@ -298,6 +298,13 @@ export class CacheInfo { ttl: Constants.oneSecond() * 6, }; + static CollectionsForAddress(address: string, pagination: QueryPagination): CacheInfo { + return { + key: `collectionsForAddress:${address}:${pagination.from}:${pagination.size}`, + ttl: Constants.oneSecond() * 6, + }; + } + static EsdtAddressesRoles(identifier: string): CacheInfo { return { key: `esdt:roles:${identifier}`, From d19f27a49aa78300061f1632ac5a170099b6edfa Mon Sep 17 00:00:00 2001 From: cfaur09 Date: Thu, 11 Dec 2025 18:34:28 +0200 Subject: [PATCH 07/10] fixes --- src/common/indexer/entities/collection.ts | 15 ++++++ .../collections/collection.service.ts | 53 +++++++++++++------ src/endpoints/esdt/esdt.address.service.ts | 3 +- 3 files changed, 54 insertions(+), 17 deletions(-) diff --git a/src/common/indexer/entities/collection.ts b/src/common/indexer/entities/collection.ts index e6a8f12dc..5190e17f6 100644 --- a/src/common/indexer/entities/collection.ts +++ b/src/common/indexer/entities/collection.ts @@ -1,3 +1,16 @@ +export interface CollectionProperties { + canMint?: boolean; + canBurn?: boolean; + canUpgrade?: boolean; + canTransferNFTCreateRole?: boolean; + canAddSpecialRoles?: boolean; + canPause?: boolean; + canFreeze?: boolean; + canWipe?: boolean; + canChangeOwner?: boolean; + canCreateMultiShard?: boolean; +} + export interface Collection { _id: string; name: string; @@ -5,9 +18,11 @@ export interface Collection { token: string; issuer: string; currentOwner: string; + numDecimals?: number; type: string; timestamp: number; ownersHistory: { address: string, timestamp: number }[]; + properties?: CollectionProperties; api_isVerified?: boolean; api_nftCount?: number; api_holderCount?: number; diff --git a/src/endpoints/collections/collection.service.ts b/src/endpoints/collections/collection.service.ts index 369831f62..30181666d 100644 --- a/src/endpoints/collections/collection.service.ts +++ b/src/endpoints/collections/collection.service.ts @@ -95,18 +95,41 @@ export class CollectionService { private async processNftCollections(tokenCollections: Collection[]): Promise { const collectionsIdentifiers = tokenCollections.map((collection) => collection.token); - const indexedCollections = new Map(); - for (const collection of tokenCollections) { - indexedCollections.set(collection.token, collection); - } + const collectionsAssets = await this.batchGetCollectionsAssets(collectionsIdentifiers); - const nftCollections: NftCollection[] = await this.applyPropertiesToCollections(collectionsIdentifiers); + const nftCollections: NftCollection[] = []; - for (const nftCollection of nftCollections) { - const indexedCollection = indexedCollections.get(nftCollection.collection); - if (indexedCollection) { - this.applyPropertiesToCollectionFromElasticSearch(nftCollection, indexedCollection); - } + for (const esCollection of tokenCollections) { + const identifierParts = esCollection.token.split('-'); + const ticker = identifierParts[0]; + const collectionBase = identifierParts.slice(0, 2).join('-'); + const assets = collectionsAssets[esCollection.token]; + const props = esCollection.properties; + + const isMetaESDT = esCollection.type === ElasticNftType.MetaESDT || esCollection.type === ElasticNftType.DynamicMetaESDT; + + const nftCollection = new NftCollection({ + name: esCollection.name, + collection: collectionBase, + ticker: ticker, + owner: esCollection.currentOwner, + assets: assets, + canFreeze: props?.canFreeze, + canWipe: props?.canWipe, + canPause: props?.canPause, + canTransferNftCreateRole: props?.canTransferNFTCreateRole, + canChangeOwner: props?.canChangeOwner, + canUpgrade: props?.canUpgrade, + canAddSpecialRoles: props?.canAddSpecialRoles, + decimals: isMetaESDT ? esCollection.numDecimals : undefined, + }); + + nftCollection.ticker = nftCollection.assets ? ticker : nftCollection.collection; + + // Apply additional ES fields (type, timestamp, counts, scamInfo) + this.applyPropertiesToCollectionFromElasticSearch(nftCollection, esCollection); + + nftCollections.push(nftCollection); } return nftCollections; @@ -146,6 +169,10 @@ export class CollectionService { } } + async buildCollectionsFromElasticData(esCollections: Collection[]): Promise { + return await this.processNftCollections(esCollections); + } + async applyPropertiesToCollections(collectionsIdentifiers: string[]): Promise { const nftCollections: NftCollection[] = []; @@ -285,17 +312,13 @@ export class CollectionService { return undefined; } - const [collection] = await this.applyPropertiesToCollections([identifier]); + const [collection] = await this.buildCollectionsFromElasticData([elasticCollection]); if (!collection) { return undefined; } const collectionDetailed = ApiUtils.mergeObjects(new NftCollectionDetailed(), collection); - collectionDetailed.type = elasticCollection.type as NftType; - collectionDetailed.timestamp = elasticCollection.timestamp; - - this.applyPropertiesToCollectionFromElasticSearch(collectionDetailed, elasticCollection); collectionDetailed.traits = await this.getCollectionTraitsCached(identifier); diff --git a/src/endpoints/esdt/esdt.address.service.ts b/src/endpoints/esdt/esdt.address.service.ts index 34ea3dc94..c0b751505 100644 --- a/src/endpoints/esdt/esdt.address.service.ts +++ b/src/endpoints/esdt/esdt.address.service.ts @@ -101,14 +101,13 @@ export class EsdtAddressService { async getCollectionsForAddress(address: string, filter: CollectionFilter, pagination: QueryPagination): Promise { const tokenCollections = await this.indexerService.getNftCollections(pagination, filter, address); - const collectionsIdentifiers = tokenCollections.map((collection) => collection.token); const indexedCollections: Record = {}; for (const collection of tokenCollections) { indexedCollections[collection.token] = collection; } - const accountCollections = await this.collectionService.applyPropertiesToCollections(collectionsIdentifiers); + const accountCollections = await this.collectionService.buildCollectionsFromElasticData(tokenCollections); const collectionsWithRoles: NftCollectionWithRoles[] = []; From edbf00762f0482bac9069cad963094cf43b7fdc0 Mon Sep 17 00:00:00 2001 From: cfaur09 Date: Wed, 17 Dec 2025 08:49:43 +0200 Subject: [PATCH 08/10] fix unit tests --- src/test/unit/services/collections.spec.ts | 55 +++------------------ src/test/unit/services/transactions.spec.ts | 4 +- 2 files changed, 11 insertions(+), 48 deletions(-) diff --git a/src/test/unit/services/collections.spec.ts b/src/test/unit/services/collections.spec.ts index 163a5441c..3d47d0e4f 100644 --- a/src/test/unit/services/collections.spec.ts +++ b/src/test/unit/services/collections.spec.ts @@ -8,11 +8,9 @@ import { PersistenceService } from "src/common/persistence/persistence.service"; import { PluginService } from "src/common/plugins/plugin.service"; import { CollectionService } from "src/endpoints/collections/collection.service"; import { CollectionFilter } from "src/endpoints/collections/entities/collection.filter"; -import { NftCollection } from "src/endpoints/collections/entities/nft.collection"; import { NftCollectionDetailed } from "src/endpoints/collections/entities/nft.collection.detailed"; import { EsdtAddressService } from "src/endpoints/esdt/esdt.address.service"; import { EsdtService } from "src/endpoints/esdt/esdt.service"; -import { NftType } from "src/endpoints/nfts/entities/nft.type"; import { CollectionRoles } from "src/endpoints/tokens/entities/collection.roles"; import { TokenAssetStatus } from "src/endpoints/tokens/entities/token.asset.status"; import { VmQueryService } from "src/endpoints/vm.query/vm.query.service"; @@ -135,6 +133,7 @@ describe('CollectionService', () => { { getTokenAssets: jest.fn(), getCollectionRanks: jest.fn(), + getAllTokenAssets: jest.fn().mockResolvedValue({}), }, }, { @@ -211,49 +210,10 @@ describe('CollectionService', () => { }); describe('getCollection', () => { - const propertiesToCollectionsMock: NftCollection = { - collection: 'XDAY23TEAM-f7a346', - type: NftType.NonFungibleESDT, - subType: undefined, - name: 'xPortalAchievements', - ticker: 'XDAY23TEAM', - owner: 'erd1lpc6wjh2hav6q50p8y6a44r2lhtnseqksygakjfgep6c9uduchkqphzu6t', - timestamp: 0, - canFreeze: true, - canWipe: true, - canPause: true, - canTransferNftCreateRole: true, - canChangeOwner: false, - canUpgrade: false, - canAddSpecialRoles: false, - decimals: undefined, - assets: { - website: 'https://xday.com', - description: - 'Test description.', - status: TokenAssetStatus.active, - pngUrl: 'https://media.elrond.com/tokens/asset/XDAY23TEAM-f7a346/logo.png', - name: '', - svgUrl: 'https://media.elrond.com/tokens/asset/XDAY23TEAM-f7a346/logo.svg', - extraTokens: [''], - ledgerSignature: '', - priceSource: undefined, - preferredRankAlgorithm: undefined, - lockedAccounts: undefined, - }, - scamInfo: undefined, - traits: [], - auctionStats: undefined, - isVerified: undefined, - holderCount: undefined, - nftCount: undefined, - }; - it('should return collection details for a given collection identifier', async () => { const identifier = 'XDAY23TEAM-f7a346'; jest.spyOn(indexerService, 'getCollection').mockResolvedValue(indexerCollectionMock); - jest.spyOn(service, 'applyPropertiesToCollections').mockResolvedValue([propertiesToCollectionsMock]); jest.spyOn(service, 'getNftCollectionRolesFromGateway').mockResolvedValue([]); const result = await service.getNftCollection(identifier); @@ -261,7 +221,6 @@ describe('CollectionService', () => { expect(result).toBeInstanceOf(Object); expect(indexerService.getCollection).toHaveBeenCalledTimes(1); expect(indexerService.getCollection).toHaveBeenCalledWith(identifier); - expect(service.applyPropertiesToCollections).toHaveBeenCalledWith([identifier]); }); it('should return undefined if the collection is not found', async () => { @@ -296,24 +255,26 @@ describe('CollectionService', () => { expect(result).toBeUndefined(); }); - it('should return undefined if no additional properties are applied to the collection', async () => { - const identifier = 'XDAY23TEAM'; + it('should return collection when ES data is available', async () => { + const identifier = 'XDAY23TEAM-f7a346'; jest.spyOn(indexerService, 'getCollection').mockResolvedValue(indexerCollectionMock); - jest.spyOn(service, 'applyPropertiesToCollections').mockResolvedValue([]); + jest.spyOn(service, 'getNftCollectionRolesFromGateway').mockResolvedValue([]); const result = await service.getNftCollection(identifier); - expect(result).toBeUndefined(); + expect(result).toBeDefined(); + expect(result?.collection).toBe('XDAY23TEAM-f7a346'); }); it('should process the collection details fully', async () => { const identifier = 'XDAY23TEAM-f7a346'; jest.spyOn(indexerService, 'getCollection').mockResolvedValue(indexerCollectionMock); - jest.spyOn(service, 'applyPropertiesToCollections').mockResolvedValue([propertiesToCollectionsMock]); jest.spyOn(service, 'getNftCollectionRolesFromGateway').mockResolvedValue([]); const result = await service.getNftCollection(identifier); expect(result).toBeInstanceOf(NftCollectionDetailed); + expect(result?.name).toBe(indexerCollectionMock.name); + expect(result?.owner).toBe(indexerCollectionMock.currentOwner); }); }); diff --git a/src/test/unit/services/transactions.spec.ts b/src/test/unit/services/transactions.spec.ts index 6a6b0c358..31f0b589c 100644 --- a/src/test/unit/services/transactions.spec.ts +++ b/src/test/unit/services/transactions.spec.ts @@ -67,7 +67,9 @@ describe('TransactionService', () => { provide: CacheService, useValue: { get: jest.fn(), - getOrSet: jest.fn(), + getOrSet: jest.fn().mockImplementation(async (_key: string, getter: () => Promise, _ttl: number) => { + return await getter(); + }), batchGetAll: jest.fn(), }, }, From c8e746a7d70b084f78a2b664f4eca532a724946e Mon Sep 17 00:00:00 2001 From: cfaur09 Date: Wed, 17 Dec 2025 10:02:23 +0200 Subject: [PATCH 09/10] fix transction filters cache --- src/endpoints/transactions/transaction.service.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/endpoints/transactions/transaction.service.ts b/src/endpoints/transactions/transaction.service.ts index be7220d02..1ca50ad62 100644 --- a/src/endpoints/transactions/transaction.service.ts +++ b/src/endpoints/transactions/transaction.service.ts @@ -877,15 +877,15 @@ export class TransactionService { return false; } - const hasNonDefaultOptions = queryOptions.withScResults || + const hasAnyEnrichmentOption = queryOptions.withScResults || queryOptions.withBlockInfo || queryOptions.withActionTransferValue || queryOptions.withUsername || queryOptions.withTxsOrder || - queryOptions.withOperations === false || - queryOptions.withLogs === false; + queryOptions.withOperations !== undefined || + queryOptions.withLogs !== undefined; - return !hasNonDefaultOptions; + return !hasAnyEnrichmentOption; } private isCacheableTransactionCount(filter: TransactionFilter, address?: string): boolean { From 2bafe6acb228927a0dec22f894b8f70b4fecc3e2 Mon Sep 17 00:00:00 2001 From: cfaur09 Date: Thu, 18 Dec 2025 13:35:59 +0200 Subject: [PATCH 10/10] fixes after review --- .../collections/collection.service.ts | 94 +++++++++---------- src/endpoints/nfts/nft.service.ts | 38 ++++---- .../transactions/transaction.service.ts | 94 +++++++++---------- 3 files changed, 112 insertions(+), 114 deletions(-) diff --git a/src/endpoints/collections/collection.service.ts b/src/endpoints/collections/collection.service.ts index 30181666d..11cbc2098 100644 --- a/src/endpoints/collections/collection.service.ts +++ b/src/endpoints/collections/collection.service.ts @@ -51,20 +51,20 @@ export class CollectionService { } async getNftCollections(pagination: QueryPagination, filter: CollectionFilter): Promise { - const executeGetCollections = async (): Promise => { - const tokenCollections = await this.indexerService.getNftCollections(pagination, filter); - return await this.processNftCollections(tokenCollections); - }; - if (this.isCacheableCollectionFilter(filter)) { return await this.cachingService.getOrSet( CacheInfo.Collections(pagination).key, - executeGetCollections, + () => this.fetchAndProcessCollections(pagination, filter), CacheInfo.Collections(pagination).ttl, ); } - return await executeGetCollections(); + return await this.fetchAndProcessCollections(pagination, filter); + } + + private async fetchAndProcessCollections(pagination: QueryPagination, filter: CollectionFilter): Promise { + const tokenCollections = await this.indexerService.getNftCollections(pagination, filter); + return await this.processNftCollections(tokenCollections); } private isCacheableCollectionFilter(filter: CollectionFilter): boolean { @@ -420,20 +420,20 @@ export class CollectionService { } async getCollectionCountForAddress(address: string, filter: CollectionFilter): Promise { - const executeCount = async () => { - const collections = await this.getCollectionsForAddress(address, filter, new QueryPagination({ from: 0, size: 10000 })); - return collections.length; - }; - if (this.isDefaultAddressCollectionFilter(filter)) { return await this.cachingService.getOrSet( CacheInfo.CollectionCountForAddress(address).key, - executeCount, + () => this.computeCollectionCountForAddress(address, filter), CacheInfo.CollectionCountForAddress(address).ttl, ); } - return await executeCount(); + return await this.computeCollectionCountForAddress(address, filter); + } + + private async computeCollectionCountForAddress(address: string, filter: CollectionFilter): Promise { + const collections = await this.getCollectionsForAddress(address, filter, new QueryPagination({ from: 0, size: 10000 })); + return collections.length; } async getCollectionForAddress(address: string, identifier: string): Promise { @@ -452,44 +452,44 @@ export class CollectionService { } async getCollectionsForAddress(address: string, filter: CollectionFilter, pagination: QueryPagination): Promise { - const executeGetCollections = async (): Promise => { - const collectionsRaw = await this.indexerService.getCollectionsForAddress(address, filter, pagination); - - if (collectionsRaw.length === 0) { - return []; - } - - const identifiers = collectionsRaw.map((x: any) => x.collection); - const collections = await this.getNftCollectionsByIds(identifiers); - - const collectionMap = new Map(); - for (const collection of collections) { - collectionMap.set(collection.collection, collection); - } - - const accountCollections: NftCollectionAccount[] = []; - for (const raw of collectionsRaw) { - const collection = collectionMap.get(raw.collection); - if (collection) { - const accountCollection = ApiUtils.mergeObjects(new NftCollectionAccount(), collection); - accountCollection.count = raw.count; - accountCollections.push(accountCollection); - } - } - - return accountCollections; - }; - if (this.isDefaultAddressCollectionFilter(filter)) { const cacheInfo = CacheInfo.CollectionsForAddress(address, pagination); return await this.cachingService.getOrSet( cacheInfo.key, - executeGetCollections, + () => this.fetchCollectionsForAddress(address, filter, pagination), cacheInfo.ttl, ); } - return await executeGetCollections(); + return await this.fetchCollectionsForAddress(address, filter, pagination); + } + + private async fetchCollectionsForAddress(address: string, filter: CollectionFilter, pagination: QueryPagination): Promise { + const collectionsRaw = await this.indexerService.getCollectionsForAddress(address, filter, pagination); + + if (collectionsRaw.length === 0) { + return []; + } + + const identifiers = collectionsRaw.map((x: any) => x.collection); + const collections = await this.getNftCollectionsByIds(identifiers); + + const collectionMap = new Map(); + for (const collection of collections) { + collectionMap.set(collection.collection, collection); + } + + const accountCollections: NftCollectionAccount[] = []; + for (const raw of collectionsRaw) { + const collection = collectionMap.get(raw.collection); + if (collection) { + const accountCollection = ApiUtils.mergeObjects(new NftCollectionAccount(), collection); + accountCollection.count = raw.count; + accountCollections.push(accountCollection); + } + } + + return accountCollections; } private isDefaultAddressCollectionFilter(filter: CollectionFilter): boolean { @@ -501,17 +501,15 @@ export class CollectionService { } async getCollectionCountForAddressWithRoles(address: string, filter: CollectionFilter): Promise { - const executeCount = async () => await this.esdtAddressService.getCollectionCountForAddressFromElastic(address, filter); - if (this.isDefaultAddressCollectionFilter(filter)) { return await this.cachingService.getOrSet( CacheInfo.CollectionRolesCountForAddress(address).key, - executeCount, + () => this.esdtAddressService.getCollectionCountForAddressFromElastic(address, filter), CacheInfo.CollectionRolesCountForAddress(address).ttl, ); } - return await executeCount(); + return await this.esdtAddressService.getCollectionCountForAddressFromElastic(address, filter); } private async getCollectionLogo(identifier: string): Promise { diff --git a/src/endpoints/nfts/nft.service.ts b/src/endpoints/nfts/nft.service.ts index d81bb7268..595721878 100644 --- a/src/endpoints/nfts/nft.service.ts +++ b/src/endpoints/nfts/nft.service.ts @@ -66,33 +66,33 @@ export class NftService { } async getNfts(queryPagination: QueryPagination, filter: NftFilter, queryOptions?: NftQueryOptions): Promise { - const executeGetNfts = async (): Promise => { - const { from, size } = queryPagination; - - const nfts = await this.getNftsInternal({ from, size }, filter); - - await Promise.all([ - this.conditionallyApplyAssetsAndTicker(nfts, undefined, queryOptions), - this.conditionallyApplyOwners(nfts, queryOptions), - this.conditionallyApplySupply(nfts, queryOptions), - this.batchProcessNfts(nfts), - ]); - - await this.batchApplyUnlockFields(nfts); - - return nfts; - }; - if (this.isCacheableNftList(filter, queryOptions)) { const cacheInfo = CacheInfo.Nfts(queryPagination); return await this.cachingService.getOrSet( cacheInfo.key, - executeGetNfts, + () => this.fetchAndProcessNfts(queryPagination, filter, queryOptions), cacheInfo.ttl, ); } - return await executeGetNfts(); + return await this.fetchAndProcessNfts(queryPagination, filter, queryOptions); + } + + private async fetchAndProcessNfts(queryPagination: QueryPagination, filter: NftFilter, queryOptions?: NftQueryOptions): Promise { + const { from, size } = queryPagination; + + const nfts = await this.getNftsInternal({ from, size }, filter); + + await Promise.all([ + this.conditionallyApplyAssetsAndTicker(nfts, undefined, queryOptions), + this.conditionallyApplyOwners(nfts, queryOptions), + this.conditionallyApplySupply(nfts, queryOptions), + this.batchProcessNfts(nfts), + ]); + + await this.batchApplyUnlockFields(nfts); + + return nfts; } private async batchProcessNfts(nfts: Nft[], fields?: string[]) { diff --git a/src/endpoints/transactions/transaction.service.ts b/src/endpoints/transactions/transaction.service.ts index 1ca50ad62..765b9429e 100644 --- a/src/endpoints/transactions/transaction.service.ts +++ b/src/endpoints/transactions/transaction.service.ts @@ -201,67 +201,67 @@ export class TransactionService { } async getTransactions(filter: TransactionFilter, pagination: QueryPagination, queryOptions?: TransactionQueryOptions, address?: string, fields?: string[]): Promise { - const computeTransactions = async (): Promise => { - const elasticTransactions = await this.indexerService.getTransactions(filter, pagination, address); + if (this.isCacheableTransactionList(filter, queryOptions, fields, address)) { + const cacheInfo = CacheInfo.Transactions(pagination); + return await this.cachingService.getOrSet( + cacheInfo.key, + () => this.computeTransactions(filter, pagination, queryOptions, address, fields), + cacheInfo.ttl, + Constants.oneSecond(), + ); + } - let transactions: TransactionDetailed[] = []; - transactions = elasticTransactions.map(x => ApiUtils.mergeObjects(new TransactionDetailed(), x)); + return await this.computeTransactions(filter, pagination, queryOptions, address, fields); + } - const hasSenderFilter = filter.sender || (filter.senders && filter.senders.length > 0); - const hasReceiverFilter = filter.receivers && filter.receivers.length > 0; + private async computeTransactions(filter: TransactionFilter, pagination: QueryPagination, queryOptions?: TransactionQueryOptions, address?: string, fields?: string[]): Promise { + const elasticTransactions = await this.indexerService.getTransactions(filter, pagination, address); - if (address && !hasSenderFilter && !hasReceiverFilter) { - transactions = this.reorderAccountSentTransactionsByNonce(transactions, address); - } + let transactions: TransactionDetailed[] = []; + transactions = elasticTransactions.map(x => ApiUtils.mergeObjects(new TransactionDetailed(), x)); - if (filter.hashes) { - const txHashes: string[] = filter.hashes; - const elasticHashes = elasticTransactions.map(({ txHash }: any) => txHash); - const missingHashes: string[] = txHashes.except(elasticHashes); + const hasSenderFilter = filter.sender || (filter.senders && filter.senders.length > 0); + const hasReceiverFilter = filter.receivers && filter.receivers.length > 0; - const gatewayTransactions = await Promise.all(missingHashes.map((txHash) => this.transactionGetService.tryGetTransactionFromGatewayForList(txHash))); - for (const gatewayTransaction of gatewayTransactions) { - if (gatewayTransaction) { - transactions.push(ApiUtils.mergeObjects(new TransactionDetailed(), gatewayTransaction)); - } - } - } + if (address && !hasSenderFilter && !hasReceiverFilter) { + transactions = this.reorderAccountSentTransactionsByNonce(transactions, address); + } - if ((queryOptions && queryOptions.withBlockInfo) || (fields && fields.includesSome(['senderBlockHash', 'receiverBlockHash', 'senderBlockNonce', 'receiverBlockNonce']))) { - await this.applyBlockInfo(transactions); - } + if (filter.hashes) { + const txHashes: string[] = filter.hashes; + const elasticHashes = elasticTransactions.map(({ txHash }: any) => txHash); + const missingHashes: string[] = txHashes.except(elasticHashes); - if (queryOptions && (queryOptions.withScResults || queryOptions.withOperations || queryOptions.withLogs)) { - queryOptions.withScResultLogs = queryOptions.withLogs; - transactions = await this.getExtraDetailsForTransactions(elasticTransactions, transactions, queryOptions); + const gatewayTransactions = await Promise.all(missingHashes.map((txHash) => this.transactionGetService.tryGetTransactionFromGatewayForList(txHash))); + for (const gatewayTransaction of gatewayTransactions) { + if (gatewayTransaction) { + transactions.push(ApiUtils.mergeObjects(new TransactionDetailed(), gatewayTransaction)); + } } + } - for (const transaction of transactions) { - transaction.type = undefined; - } + if ((queryOptions && queryOptions.withBlockInfo) || (fields && fields.includesSome(['senderBlockHash', 'receiverBlockHash', 'senderBlockNonce', 'receiverBlockNonce']))) { + await this.applyBlockInfo(transactions); + } - await this.processTransactions(transactions, { - withScamInfo: queryOptions?.withScamInfo ?? false, - withUsername: queryOptions?.withUsername ?? false, - withActionTransferValue: queryOptions?.withActionTransferValue ?? false, - }); + if (queryOptions && (queryOptions.withScResults || queryOptions.withOperations || queryOptions.withLogs)) { + queryOptions.withScResultLogs = queryOptions.withLogs; + transactions = await this.getExtraDetailsForTransactions(elasticTransactions, transactions, queryOptions); + } - this.processRelayedInfo(transactions); + for (const transaction of transactions) { + transaction.type = undefined; + } - return transactions; - }; + await this.processTransactions(transactions, { + withScamInfo: queryOptions?.withScamInfo ?? false, + withUsername: queryOptions?.withUsername ?? false, + withActionTransferValue: queryOptions?.withActionTransferValue ?? false, + }); - if (this.isCacheableTransactionList(filter, queryOptions, fields, address)) { - const cacheInfo = CacheInfo.Transactions(pagination); - return await this.cachingService.getOrSet( - cacheInfo.key, - computeTransactions, - cacheInfo.ttl, - Constants.oneSecond(), - ); - } + this.processRelayedInfo(transactions); - return await computeTransactions(); + return transactions; } private getAssetsFromUsername(username: string | null | undefined): AccountAssets | undefined {