Tokens Service Performance Improvements

src/crons/cache.warmer/cache.warmer.service.ts

@@ -26,7 +26,7 @@ import { IndexerService } from "src/common/indexer/indexer.service";
 import { NftService } from "src/endpoints/nfts/nft.service";
 import { AccountQueryOptions } from "src/endpoints/accounts/entities/account.query.options";
 import { Account, TokenType } from "src/common/indexer/entities";
-import { TokenDetailed } from "src/endpoints/tokens/entities/token.detailed";
+import { ArrayIndexer } from "src/utils/array.indexer";
 import { DataApiService } from "src/common/data-api/data-api.service";
 import { BlockService } from "src/endpoints/blocks/block.service";
 import { PoolService } from "src/endpoints/pool/pool.service";
@@ -277,10 +277,9 @@ export class CacheWarmerService {
   async handleTokenAssetsExtraInfoInvalidations() {
     const assets = await this.assetsService.getAllTokenAssets();
     const allTokens = await this.tokenService.getAllTokens();
-    const allTokensIndexed = allTokens.toRecord<TokenDetailed>(token => token.identifier);
 
     for (const identifier of Object.keys(assets)) {
-      const token = allTokensIndexed[identifier];
+      const token = ArrayIndexer.getItemByKeyValue(allTokens, 'identifier', identifier);
       if (!token) {
         continue;
       }

src/endpoints/tokens/token.service.ts

@@ -29,6 +29,7 @@ import { TokenLogo } from "./entities/token.logo";
 import { AssetsService } from "src/common/assets/assets.service";
 import { CacheInfo } from "src/utils/cache.info";
 import { TokenAssets } from "src/common/assets/entities/token.assets";
+import { ArrayIndexer } from "src/utils/array.indexer";
 import { TransactionFilter } from "../transactions/entities/transaction.filter";
 import { TransactionService } from "../transactions/transaction.service";
 import { MexTokenService } from "../mex/mex.token.service";
@@ -74,14 +75,13 @@ export class TokenService {
 
   async isToken(identifier: string): Promise<boolean> {
     const tokens = await this.getAllTokens();
-    const lowercaseIdentifier = identifier.toLowerCase();
-    return tokens.find(x => x.identifier.toLowerCase() === lowercaseIdentifier) !== undefined;
+    return ArrayIndexer.getItemByKeyValue(tokens, 'identifier', this.normalizeIdentifierCase(identifier)) !== undefined;
   }
 
   async getToken(rawIdentifier: string, supplyOptions?: TokenSupplyOptions): Promise<TokenDetailed | undefined> {
     const tokens = await this.getAllTokens();
     const identifier = this.normalizeIdentifierCase(rawIdentifier);
-    let token = tokens.find(x => x.identifier === identifier);
+    let token = ArrayIndexer.getItemByKeyValue(tokens, 'identifier', identifier);
 
     if (!TokenUtils.isToken(identifier)) {
       return undefined;
@@ -148,55 +148,57 @@ export class TokenService {
   async getFilteredTokens(filter: TokenFilter): Promise<TokenDetailed[]> {
     let tokens = await this.getAllTokens();
 
-    if (filter.type) {
-      tokens = tokens.filter(token => token.type === filter.type);
-    }
-
-    if (filter.subType) {
-      tokens = tokens.filter(token => token.subType.toString() === filter.subType?.toString());
-    }
+    // Precompute filters only once per request
+    const mexPairTypes = filter.mexPairType ?? [];
+    const searchLower = filter.search?.toLowerCase();
+    const nameLower = filter.name?.toLowerCase();
+    const identifierLower = filter.identifier?.toLowerCase();
+    const identifiersLower = filter.identifiers?.map(identifier => identifier.toLowerCase());
+
+    tokens = tokens.filter(token => {
+      if (filter.type && token.type !== filter.type) {
+        return false;
+      }
 
-    if (filter.search) {
-      const searchLower = filter.search.toLowerCase();
+      if (filter.subType && token.subType.toString() !== filter.subType.toString()) {
+        return false;
+      }
 
-      tokens = tokens.filter(token => token.name.toLowerCase().includes(searchLower) || token.identifier.toLowerCase().includes(searchLower));
-    }
+      if (searchLower && !token.name.toLowerCase().includes(searchLower) && !token.identifier.toLowerCase().includes(searchLower)) {
+        return false;
+      }
 
-    if (filter.name) {
-      const nameLower = filter.name.toLowerCase();
+      if (nameLower && token.name.toLowerCase() !== nameLower) {
+        return false;
+      }
 
-      tokens = tokens.filter(token => token.name.toLowerCase() === nameLower);
-    }
+      if (identifierLower && !token.identifier.toLowerCase().includes(identifierLower)) {
+        return false;
+      }
 
-    if (filter.identifier) {
-      const identifierLower = filter.identifier.toLowerCase();
+      if (identifiersLower && !identifiersLower.includes(token.identifier.toLowerCase())) {
+        return false;
+      }
 
-      tokens = tokens.filter(token => token.identifier.toLowerCase().includes(identifierLower));
-    }
+      if (filter.includeMetaESDT !== true && token.type !== TokenType.FungibleESDT) {
+        return false;
+      }
 
-    if (filter.identifiers) {
-      const identifierArray = filter.identifiers.map(identifier => identifier.toLowerCase());
+      if (mexPairTypes.length > 0 && !mexPairTypes.includes(token.mexPairType)) {
+        return false;
+      }
 
-      tokens = tokens.filter(token => identifierArray.includes(token.identifier.toLowerCase()));
-    }
+      if (filter.priceSource && token.assets?.priceSource?.type !== filter.priceSource) {
+        return false;
+      }
 
-    if (filter.includeMetaESDT !== true) {
-      tokens = tokens.filter(token => token.type === TokenType.FungibleESDT);
-    }
+      return true;
+    });
 
     if (filter.sort) {
       tokens = this.sortTokens(tokens, filter.sort, filter.order ?? SortOrder.desc);
     }
 
-    const mexPairTypes = filter.mexPairType ?? [];
-    if (mexPairTypes.length > 0) {
-      tokens = tokens.filter(token => mexPairTypes.includes(token.mexPairType));
-    }
-
-    if (filter.priceSource) {
-      tokens = tokens.filter(token => token.assets?.priceSource?.type === filter.priceSource);
-    }
-
     return tokens;
   }
 
@@ -278,12 +280,10 @@ export class TokenService {
 
     const allTokens = await this.getAllTokens();
 
-    const allTokensIndexed = allTokens.toRecord<TokenDetailed>(token => token.identifier);
-
     const result: TokenWithBalance[] = [];
     for (const elasticToken of elasticTokens) {
-      if (allTokensIndexed[elasticToken.token]) {
-        const token = allTokensIndexed[elasticToken.token];
+      const token = ArrayIndexer.getItemByKeyValue(allTokens, 'identifier', elasticToken.token);
+      if (token) {
 
         const tokenWithBalance: TokenWithBalance = {
           ...token,
@@ -658,7 +658,7 @@ export class TokenService {
     const result: TokenWithRoles[] = [];
 
     for (const item of tokenList) {
-      const token = allTokens.find(x => x.identifier === item.identifier);
+      const token = ArrayIndexer.getItemByKeyValue(allTokens, 'identifier', item.identifier);
       if (token) {
         this.applyTickerFromAssets(token);
 

src/test/unit/utils/array.indexer.spec.ts

@@ -0,0 +1,75 @@
+import { ArrayIndexer } from "src/utils/array.indexer";
+
+describe('ArrayIndexer', () => {
+  const tokenItems = [
+    { identifier: 'AAA-123', nonce: 1, ticker: 'AAA' },
+    { identifier: 'BBB-456', nonce: 2, ticker: 'BBB' },
+    { identifier: 'CCC-789', nonce: 3, ticker: 'CCC' },
+  ];
+
+  describe('getOrSetPositions', () => {
+    it('builds positions map for a property', () => {
+      const positions = ArrayIndexer.getOrSetPositions(tokenItems, 'identifier');
+
+      expect(positions['AAA-123']).toStrictEqual(0);
+      expect(positions['BBB-456']).toStrictEqual(1);
+      expect(positions['CCC-789']).toStrictEqual(2);
+    });
+
+    it('reuses cached positions map for same array and property', () => {
+      const first = ArrayIndexer.getOrSetPositions(tokenItems, 'identifier');
+      const second = ArrayIndexer.getOrSetPositions(tokenItems, 'identifier');
+
+      expect(second).toBe(first);
+    });
+
+    it('builds independent maps for different properties', () => {
+      const byIdentifier = ArrayIndexer.getOrSetPositions(tokenItems, 'identifier');
+      const byTicker = ArrayIndexer.getOrSetPositions(tokenItems, 'ticker');
+
+      expect(byIdentifier).not.toBe(byTicker);
+      expect(byTicker['AAA']).toStrictEqual(0);
+      expect(byTicker['BBB']).toStrictEqual(1);
+      expect(byTicker['CCC']).toStrictEqual(2);
+    });
+
+    it('does not reuse cache for a different array instance', () => {
+      const firstArrayInstance = [
+        { identifier: 'AAA-123', nonce: 1, ticker: 'AAA' },
+        { identifier: 'BBB-456', nonce: 2, ticker: 'BBB' },
+        { identifier: 'CCC-789', nonce: 3, ticker: 'CCC' },
+      ];
+      const secondArrayInstance = [
+        { identifier: 'AAA-123', nonce: 1, ticker: 'AAA' },
+        { identifier: 'BBB-456', nonce: 2, ticker: 'BBB' },
+        { identifier: 'CCC-789', nonce: 3, ticker: 'CCC' },
+      ];
+
+      const first = ArrayIndexer.getOrSetPositions(firstArrayInstance, 'identifier');
+      const second = ArrayIndexer.getOrSetPositions(secondArrayInstance, 'identifier');
+
+      expect(second).not.toBe(first);
+      expect(second).toStrictEqual(first);
+    });
+  });
+
+  describe('getItemByKeyValue', () => {
+    it('returns item for existing string key', () => {
+      const result = ArrayIndexer.getItemByKeyValue(tokenItems, 'identifier', 'BBB-456');
+
+      expect(result).toStrictEqual(tokenItems[1]);
+    });
+
+    it('returns item for existing numeric key', () => {
+      const result = ArrayIndexer.getItemByKeyValue(tokenItems, 'nonce', 3);
+
+      expect(result).toStrictEqual(tokenItems[2]);
+    });
+
+    it('returns undefined for missing key', () => {
+      const result = ArrayIndexer.getItemByKeyValue(tokenItems, 'identifier', 'MISSING');
+
+      expect(result).toBeUndefined();
+    });
+  });
+});

src/utils/array.indexer.ts

@@ -0,0 +1,76 @@
+/**
+ * Utility class designed to provide O(1) lookups for arrays of objects.
+ * It uses a WeakMap to cache the positions of elements based on a specific property.
+ * The WeakMap ensures that once the array is garbage collected, its associated cache is also cleared, preventing memory leaks.
+ */
+export class ArrayIndexer {
+  /**
+   * The cache stores the array instance as the WeakMap key.
+   * The value is an internal Map where:
+   * - Key: The name of the property used for indexing (e.g., 'identifier').
+   * - Value: A Record mapping the actual property values to their index in the array.
+   */
+  private static readonly cache = new WeakMap<any[], Map<string, Record<string, number>>>();
+
+  /**
+   * Retrieves or builds a dictionary (Record) of array indices based on a specified property.
+   * * @param array The array instance to be indexed.
+   * @param propertyNameForKey The property key of the objects inside the array used to build the index.
+   * @returns A Record mapping the property values to their corresponding indices in the array.
+   */
+  static getOrSetPositions<T>(array: T[], propertyNameForKey: keyof T): Record<string, number> {
+    const propertyString = String(propertyNameForKey);
+
+    // 1. Check if we already have cache entries for this specific array instance
+    let arrayRecords = this.cache.get(array);
+
+    if (arrayRecords) {
+      // 2. Check if we already computed the index Record for this specific property
+      const cachedRecord = arrayRecords.get(propertyString);
+      if (cachedRecord) {
+        return cachedRecord; // Cache HIT: Return the existing index map
+      }
+    } else {
+      // Initialize the internal Map for this new array instance
+      arrayRecords = new Map<string, Record<string, number>>();
+      this.cache.set(array, arrayRecords);
+    }
+
+    // 3. Cache MISS: Build the index Record by iterating through the array O(N)
+    const record: Record<string, number> = {};
+
+    for (let index = 0; index < array.length; index++) {
+      const element = array[index];
+      const key = String(element[propertyNameForKey]);
+
+      // Map the property's stringified value to its position (index) in the array
+      record[key] = index;
+    }
+
+    // 4. Save the newly built Record into the cache map for future use
+    arrayRecords.set(propertyString, record);
+
+    return record;
+  }
+
+  /**
+   * Quickly retrieves an item from the array using the specified property and its value.
+   * Leverages the cached index Record to perform an O(1) lookup.
+   * * @param array The array to search in.
+   * @param propertyNameForKey The property used for matching.
+   * @param searchedKeyValue The exact value of the property to find.
+   * @returns The found element, or undefined if it doesn't exist.
+   */
+  static getItemByKeyValue<T>(array: T[], propertyNameForKey: keyof T, searchedKeyValue: string | number): T | undefined {
+    // Retrieve the cached index mapping for this array and property
+    const index = this.getOrSetPositions(array, propertyNameForKey)[String(searchedKeyValue)];
+
+    // If the index doesn't exist in our map, the item is not in the array
+    if (index === undefined) {
+      return undefined;
+    }
+
+    // Return the element directly from the array using the fast O(1) index lookup
+    return array[index];
+  }
+}