add configurable response compression with gzip deflate support + update package json (#1512)

* add configurable response compresion

* set true compression for testnet env

* refactor processNFtCollections method

* update package

* update version of node

* update unit test yaml

* getNfts parallel execution

* compression flag false by default

* improve getSingleNFt

* getNftsCollection increase performance

* undo last commit

Author: cfaur09

.github/workflows/lint.yml

@@ -15,7 +15,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [16.x]
+        node-version: [18.x]
         # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
 
     steps:

.github/workflows/unit.tests.yml

@@ -15,7 +15,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [16.x]
+        node-version: [18.x]
         # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
 
     steps:

config/config.devnet.yaml

@@ -177,3 +177,8 @@ inflation:
 nftProcess:
   parallelism: 1
   maxRetries: 3
+compression:
+  enabled: true
+  level: 6
+  threshold: 1024
+  chunkSize: 16384

config/config.mainnet.yaml

@@ -181,3 +181,8 @@ inflation:
 nftProcess:
   parallelism: 1
   maxRetries: 3
+compression:
+  enabled: true
+  level: 6
+  threshold: 1024
+  chunkSize: 16384

config/config.testnet.yaml

@@ -180,3 +180,8 @@ inflation:
 nftProcess:
   parallelism: 1
   maxRetries: 3
+compression:
+  enabled: true
+  level: 6
+  threshold: 1024
+  chunkSize: 16384

package-lock.json

@@ -119,9 +119,10 @@
     "apollo-server-core": "^3.13.0",
     "apollo-server-express": "3.13.0",
     "bignumber.js": "^9.0.2",
+    "compression": "^1.8.0",
     "crypto-js": "^4.1.1",
-    "dataloader": "^2.1.0",
-    "dd-trace": "^1.6.0",
+    "dataloader": "^2.2.2",
+    "dd-trace": "5.56.0",
     "fluent-ffmpeg": "^2.1.2",
     "graphql": "^16.8.1",
     "graphql-fields-list": "^2.2.4",
@@ -141,7 +142,7 @@
     "redis": "^3.1.2",
     "reflect-metadata": "^0.1.13",
     "request-ip": "^2.1.3",
-    "rimraf": "^3.0.2",
+    "rimraf": "^5.0.0",
     "rxjs": "^7.1.0",
     "sharp": "^0.32.6",
     "simple-git": "^3.16.0",
@@ -160,6 +161,7 @@
     "@nestjs/schematics": "10.0.2",
     "@nestjs/testing": "10.2.4",
     "@testing-library/jest-dom": "6.1.4",
+    "@types/compression": "^1.8.1",
     "@types/cron": "^1.7.3",
     "@types/crypto-js": "^4.0.2",
     "@types/express": "^4.17.13",

package.json

@@ -934,4 +934,20 @@ export class ApiConfigService {
   getCacheDuration(): number {
     return this.configService.get<number>('caching.cacheDuration') ?? 3;
   }
+
+  getCompressionEnabled(): boolean {
+    return this.configService.get<boolean>('compression.enabled') ?? false;
+  }
+
+  getCompressionLevel(): number {
+    return this.configService.get<number>('compression.level') ?? 6;
+  }
+
+  getCompressionThreshold(): number {
+    return this.configService.get<number>('compression.threshold') ?? 1024;
+  }
+
+  getCompressionChunkSize(): number {
+    return this.configService.get<number>('compression.chunkSize') ?? 16384;
+  }
 }

src/common/api-config/api.config.service.ts

@@ -69,44 +69,92 @@ export class NftService {
 
     const nfts = await this.getNftsInternal({ from, size }, filter);
 
-    for (const nft of nfts) {
-      await this.applyAssetsAndTicker(nft);
-    }
+    await Promise.all([
+      this.batchApplyAssetsAndTicker(nfts),
+      this.conditionallyApplyOwners(nfts, queryOptions),
+      this.conditionallyApplySupply(nfts, queryOptions),
+      this.batchProcessNfts(nfts),
+    ]);
 
-    if (queryOptions && queryOptions.withOwner) {
-      const nftsIdentifiers = nfts.filter(x => x.type === NftType.NonFungibleESDT).map(x => x.identifier);
+    await this.batchApplyUnlockFields(nfts);
 
-      const accountsEsdts = await this.getAccountEsdtByIdentifiers(nftsIdentifiers, { from: 0, size: nftsIdentifiers.length });
+    return nfts;
+  }
 
-      for (const nft of nfts) {
-        if (nft.type === NftType.NonFungibleESDT) {
-          const accountEsdt = accountsEsdts.find((accountEsdt: any) => accountEsdt.identifier == nft.identifier);
-          if (accountEsdt) {
-            nft.owner = accountEsdt.address;
-          }
+  private async batchProcessNfts(nfts: Nft[], fields?: string[]) {
+    await Promise.all([
+      this.batchApplyMedia(nfts, fields),
+      this.batchApplyMetadata(nfts, fields),
+    ]);
+  }
+
+  private async batchApplyAssetsAndTicker(nfts: Nft[], fields?: string[]): Promise<void> {
+    if (fields && fields.includesNone(['ticker', 'assets'])) {
+      return;
+    }
+
+    await Promise.all(
+      nfts.map(async (nft) => {
+        nft.assets = await this.assetsService.getTokenAssets(nft.identifier) ??
+          await this.assetsService.getTokenAssets(nft.collection);
+
+        if (nft.assets) {
+          nft.ticker = nft.collection.split('-')[0];
+        } else {
+          nft.ticker = nft.collection;
         }
-      }
+      })
+    );
+  }
+
+  private async conditionallyApplyOwners(nfts: Nft[], queryOptions?: NftQueryOptions): Promise<void> {
+    if (!queryOptions?.withOwner) {
+      return;
     }
 
-    if (queryOptions && queryOptions.withSupply) {
-      const supplyNfts = nfts.filter(nft => nft.type.in(NftType.SemiFungibleESDT, NftType.MetaESDT));
-      await this.batchApplySupply(supplyNfts);
+    const nftsIdentifiers = nfts.filter(x => x.type === NftType.NonFungibleESDT).map(x => x.identifier);
+
+    if (nftsIdentifiers.length === 0) {
+      return;
     }
 
-    await this.batchProcessNfts(nfts);
+    const accountsEsdts = await this.getAccountEsdtByIdentifiers(nftsIdentifiers, {
+      from: 0,
+      size: nftsIdentifiers.length,
+    });
+
+    const ownerMap = accountsEsdts.reduce((acc: Record<string, string>, accountEsdt: any) => {
+      acc[accountEsdt.identifier] = accountEsdt.address;
+      return acc;
+    }, {});
 
     for (const nft of nfts) {
-      await this.applyUnlockFields(nft);
+      if (nft.type === NftType.NonFungibleESDT && ownerMap[nft.identifier]) {
+        nft.owner = ownerMap[nft.identifier];
+      }
+    }
+  }
+
+  private async conditionallyApplySupply(nfts: Nft[], queryOptions?: NftQueryOptions): Promise<void> {
+    if (!queryOptions?.withSupply) {
+      return;
     }
 
-    return nfts;
+    const supplyNfts = nfts.filter(nft => nft.type.in(NftType.SemiFungibleESDT, NftType.MetaESDT));
+
+    if (supplyNfts.length > 0) {
+      await this.batchApplySupply(supplyNfts);
+    }
   }
 
-  private async batchProcessNfts(nfts: Nft[], fields?: string[]) {
-    await Promise.all([
-      this.batchApplyMedia(nfts, fields),
-      this.batchApplyMetadata(nfts, fields),
-    ]);
+  private async batchApplyUnlockFields(nfts: Nft[], fields?: string[]): Promise<void> {
+    if (fields && fields.includesNone(['unlockSchedule', 'unlockEpoch'])) {
+      return;
+    }
+
+    await Promise.all(
+      nfts.map(nft => this.applyUnlockFields(nft, fields))
+    );
   }
 
   private async applyNftOwner(nft: Nft): Promise<void> {
@@ -219,23 +267,29 @@ export class NftService {
       return undefined;
     }
 
-    if (nft.type && nft.type.in(
-      NftType.SemiFungibleESDT, NftType.MetaESDT,
-      NftSubType.DynamicSemiFungibleESDT, NftSubType.DynamicMetaESDT
-    )) {
-      await this.applySupply(nft);
-    }
-
-    await this.applyNftOwner(nft);
-
-    await this.applyNftAttributes(nft);
+    const types = [
+      NftType.SemiFungibleESDT,
+      NftType.MetaESDT,
+      NftSubType.DynamicSemiFungibleESDT,
+      NftSubType.DynamicMetaESDT,
+    ];
 
-    await this.applyAssetsAndTicker(nft);
+    await Promise.all([
+      (async () => {
+        if (nft.type && types.includes(nft.type as NftType | NftSubType)) {
+          await this.applySupply(nft);
+        }
+      })(),
+      this.applyAssetsAndTicker(nft),
+      this.processNft(nft),
+      (async () => {
+        await this.applyNftOwner(nft);
+        await this.applyNftAttributes(nft);
+      })(),
+    ]);
 
     await this.applyUnlockFields(nft);
 
-    await this.processNft(nft);
-
     return nft;
   }
 

src/endpoints/nfts/nft.service.ts

@@ -8,8 +8,6 @@ import { PrivateAppModule } from './private.app.module';
 import { CacheWarmerModule } from './crons/cache.warmer/cache.warmer.module';
 import { MicroserviceOptions, Transport } from '@nestjs/microservices';
 import { INestApplication, Logger, NestInterceptor } from '@nestjs/common';
-import * as bodyParser from 'body-parser';
-import * as requestIp from 'request-ip';
 import { WINSTON_MODULE_NEST_PROVIDER } from 'nest-winston';
 import { RedisClient } from 'redis';
 import { TransactionProcessorModule } from './crons/transaction.processor/transaction.processor.module';
@@ -35,6 +33,9 @@ import { JwtOrNativeAuthGuard } from '@multiversx/sdk-nestjs-auth';
 import { WebSocketPublisherModule } from './common/websockets/web-socket-publisher-module';
 import { IndexerService } from './common/indexer/indexer.service';
 import { NotWritableError } from './common/indexer/entities/not.writable.error';
+import * as bodyParser from 'body-parser';
+import * as requestIp from 'request-ip';
+import compression from 'compression';
 
 async function bootstrap() {
   const logger = new Logger('Bootstrapper');
@@ -178,6 +179,21 @@ async function bootstrap() {
 }
 
 async function configurePublicApp(publicApp: NestExpressApplication, apiConfigService: ApiConfigService) {
+  if (apiConfigService.getCompressionEnabled()) {
+    publicApp.use(compression({
+      filter: (req: any, res: any) => {
+        if (req.headers['x-no-compression']) {
+          return false;
+        }
+        return compression.filter(req, res);
+      },
+      level: apiConfigService.getCompressionLevel(),
+      threshold: apiConfigService.getCompressionThreshold(),
+      memLevel: 8,
+      chunkSize: apiConfigService.getCompressionChunkSize(),
+    }));
+  }
+
   publicApp.use(bodyParser.json({ limit: '1mb' }));
   publicApp.use(requestIp.mw());
   publicApp.enableCors();
@@ -297,6 +313,10 @@ async function configurePublicApp(publicApp: NestExpressApplication, apiConfigSe
   logger.log(`Use request caching: ${await settingsService.getUseRequestCachingFlag()}`);
   logger.log(`Use request logging: ${await settingsService.getUseRequestLoggingFlag()}`);
   logger.log(`Use vm query tracing: ${await settingsService.getUseVmQueryTracingFlag()}`);
+  logger.log(`Compression enabled: ${apiConfigService.getCompressionEnabled()}`);
+  if (apiConfigService.getCompressionEnabled()) {
+    logger.log(`Compression level: ${apiConfigService.getCompressionLevel()} (threshold: ${apiConfigService.getCompressionThreshold()} bytes)`);
+  }
 }
 
 async function configureCacheWarmerApp(cacheWarmerApp: INestApplication<any>, apiConfigService: ApiConfigService): Promise<void> {