Merge pull request #144 from topcoder-platform/dev

Export optimization (PS-530)

Author: jmgasper

src/api/admin/admin.controller.ts

@@ -9,6 +9,7 @@ import {
   HttpCode,
   HttpStatus,
   BadRequestException,
+  InternalServerErrorException,
 } from '@nestjs/common';
 import {
   ApiOperation,
@@ -37,6 +38,8 @@ import { AccessControlService } from 'src/shared/access-control';
 @Controller('/admin')
 @ApiBearerAuth()
 export class AdminController {
+  private static readonly EXPORT_BATCH_SIZE = 1000;
+
   constructor(
     private readonly adminService: AdminService,
     private readonly winningsRepo: WinningsRepository,
@@ -113,22 +116,53 @@ export class AdminController {
   @Header('Content-Type', 'text/csv')
   @Header('Content-Disposition', 'attachment; filename="winnings.csv"')
   async exportWinnings(@Body() body: WinningRequestDto, @User() user: any) {
-    const filters =
+    const baseFilters =
       await this.accessControlService.applyFilters<WinningRequestDto>(
         user.id,
         user.roles,
         {
           ...body,
-          limit: 999,
+          limit: undefined,
+          offset: undefined,
         },
       );
-    const result = await this.winningsRepo.searchWinnings(filters);
+
+    const winnings: SearchWinningResult['winnings'] = [];
+    let offset = 0;
+
+    while (true) {
+      const result = await this.winningsRepo.searchWinnings(
+        {
+          ...baseFilters,
+          limit: AdminController.EXPORT_BATCH_SIZE,
+          offset,
+        },
+        {
+          includeCount: false,
+          includePayoutStatus: false,
+        },
+      );
+
+      if (result.error || !result.data) {
+        throw new InternalServerErrorException(
+          result.error?.message ?? 'Export winnings failed',
+        );
+      }
+
+      winnings.push(...result.data.winnings);
+
+      if (result.data.winnings.length < AdminController.EXPORT_BATCH_SIZE) {
+        break;
+      }
+
+      offset += AdminController.EXPORT_BATCH_SIZE;
+    }
 
     const handles = await this.tcMembersService.getHandlesByUserIds(
-      result.data.winnings.map((d) => d.winnerId),
+      winnings.map((d) => d.winnerId),
     );
 
-    const csvRes = result.data.winnings.map((item) => {
+    const csvRes = winnings.map((item) => {
       const payment =
         item.details && item.details.length > 0 ? item.details[0] : null;
 

src/api/repository/winnings.repo.ts

@@ -21,6 +21,11 @@ import { Logger } from 'src/shared/global';
 
 const ONE_DAY = 24 * 60 * 60 * 1000;
 
+interface SearchWinningsOptions {
+  includeCount?: boolean;
+  includePayoutStatus?: boolean;
+}
+
 @Injectable()
 export class WinningsRepository {
   private readonly logger = new Logger(WinningsRepository.name);
@@ -162,8 +167,11 @@ export class WinningsRepository {
    */
   async searchWinnings(
     searchProps: WinningRequestDto,
+    options: SearchWinningsOptions = {},
   ): Promise<ResponseDto<SearchWinningResult>> {
     const result = new ResponseDto<SearchWinningResult>();
+    const includeCount = options.includeCount ?? true;
+    const includePayoutStatus = options.includePayoutStatus ?? true;
 
     try {
       let winnerIds: string[] | undefined;
@@ -202,7 +210,9 @@ export class WinningsRepository {
         !winnerIds && !!externalIds?.length,
       );
 
-      const winnings = await this.prisma.winnings.findMany({
+      const limit = searchProps.limit ?? 10;
+      const offset = searchProps.offset ?? 0;
+      const winningsPromise = this.prisma.winnings.findMany({
         where: queryWhere,
         include: {
           payment: {
@@ -218,15 +228,22 @@ export class WinningsRepository {
           origin: true,
         },
         orderBy,
-        skip: searchProps.offset,
-        take: searchProps.limit,
+        skip: offset,
+        take: limit,
       });
 
-      const count = await this.prisma.winnings.count({ where: queryWhere });
+      const [winnings, count] = includeCount
+        ? await Promise.all([
+            winningsPromise,
+            this.prisma.winnings.count({ where: queryWhere }),
+          ])
+        : [await winningsPromise, 0];
 
-      const usersPayoutStatusMap = winnings?.length
-        ? await this.getUsersPayoutStatusForWinnings(winnings)
-        : ({} as { [key: string]: payment_status });
+      const usersPayoutStatusMap: Record<string, unknown> =
+        includePayoutStatus && winnings?.length
+          ? await this.getUsersPayoutStatusForWinnings(winnings)
+          : {};
+      const totalItems = includeCount ? count : winnings.length;
 
       result.data = {
         winnings: winnings.map((item) => ({
@@ -257,13 +274,15 @@ export class WinningsRepository {
             item.payment?.[0].updated_at ??
             undefined) as Date,
           releaseDate: item.payment?.[0]?.release_date as Date,
-          paymentStatus: usersPayoutStatusMap[item.winner_id],
+          paymentStatus: usersPayoutStatusMap[
+            item.winner_id
+          ] as WinningDto['paymentStatus'],
         })),
         pagination: {
-          totalItems: count,
-          totalPages: Math.ceil(count / searchProps.limit),
-          pageSize: searchProps.limit,
-          currentPage: Math.ceil(searchProps.offset / searchProps.limit) + 1,
+          totalItems,
+          totalPages: Math.ceil(totalItems / limit),
+          pageSize: limit,
+          currentPage: Math.ceil(offset / limit) + 1,
         },
       };
       // response.data = winnings as any

src/shared/access-control/engagement-pa.provider.ts

@@ -30,7 +30,7 @@ export class EngagementPaymentApproverProvider implements RoleAccessProvider {
     );
     if (unauthorized.length > 0) {
       throw new Error(
-        `${Role.EngagementPaymentApprover} user is trying to access winning with category='${unauthorized.map(w => w.category).join(', ')}'`,
+        `${Role.EngagementPaymentApprover} user is trying to access winning with category='${unauthorized.map((w) => w.category).join(', ')}'`,
       );
     }
   }