Optimizations for export (PS-530)

Author: jmgasper

src/api/admin/admin.controller.ts

@@ -9,6 +9,7 @@ import {
   HttpCode,
   HttpStatus,
   BadRequestException,
+  InternalServerErrorException,
 } from '@nestjs/common';
 import {
   ApiOperation,
@@ -38,6 +39,8 @@ import { WinningUpdateRequestDto } from './dto/winnings.dto';
 @Controller('/admin')
 @ApiBearerAuth()
 export class AdminController {
+  private static readonly EXPORT_BATCH_SIZE = 1000;
+
   constructor(
     private readonly adminService: AdminService,
     private readonly winningsRepo: WinningsRepository,
@@ -118,22 +121,52 @@ export class AdminController {
   @Header('Content-Type', 'text/csv')
   @Header('Content-Disposition', 'attachment; filename="winnings.csv"')
   async exportWinnings(@Body() body: WinningRequestDto, @User() user: any) {
-    const result = await this.winningsRepo.searchWinnings(
-      await this.adminService.applyBaAdminUserFilters(
-        user.id,
-        this.isBaAdmin(user),
+    const baseFilters = await this.adminService.applyBaAdminUserFilters(
+      user.id,
+      this.isBaAdmin(user),
+      {
+        ...body,
+        limit: undefined,
+        offset: undefined,
+      },
+    );
+
+    const winnings: SearchWinningResult['winnings'] = [];
+    let offset = 0;
+
+    while (true) {
+      const result = await this.winningsRepo.searchWinnings(
         {
-          ...body,
-          limit: 999,
+          ...baseFilters,
+          limit: AdminController.EXPORT_BATCH_SIZE,
+          offset,
         },
-      ),
-    );
+        {
+          includeCount: false,
+          includePayoutStatus: false,
+        },
+      );
+
+      if (result.error || !result.data) {
+        throw new InternalServerErrorException(
+          result.error?.message ?? 'Export winnings failed',
+        );
+      }
+
+      winnings.push(...result.data.winnings);
+
+      if (result.data.winnings.length < AdminController.EXPORT_BATCH_SIZE) {
+        break;
+      }
+
+      offset += AdminController.EXPORT_BATCH_SIZE;
+    }
 
     const handles = await this.tcMembersService.getHandlesByUserIds(
-      result.data.winnings.map((d) => d.winnerId),
+      winnings.map((d) => d.winnerId),
     );
 
-    const csvRes = result.data.winnings.map((item) => {
+    const csvRes = winnings.map((item) => {
       const payment =
         item.details && item.details.length > 0 ? item.details[0] : null;
 

src/api/admin/admin.service.ts

@@ -49,10 +49,10 @@ export class AdminService {
     const baRows = await baPrisma.billingAccountAccess.findMany({
       where: {
         userId,
-      }
+      },
     });
 
-    return baRows.map(r => `${r.billingAccountId}`);
+    return baRows.map((r) => `${r.billingAccountId}`);
   }
 
   async applyBaAdminUserFilters(
@@ -109,8 +109,8 @@ export class AdminService {
       },
       select: {
         billing_account: true,
-      }
-    });;
+      },
+    });
 
     if (!payments || payments.length === 0) {
       // nothing to check
@@ -173,8 +173,6 @@ export class AdminService {
         throw new NotFoundException('failed to get current payments');
       }
 
-
-
       let releaseDate;
       if (body.paymentStatus) {
         releaseDate = await this.getPaymentReleaseDateByWinningsId(winningsId);

src/api/repository/winnings.repo.ts

@@ -21,6 +21,11 @@ import { Logger } from 'src/shared/global';
 
 const ONE_DAY = 24 * 60 * 60 * 1000;
 
+interface SearchWinningsOptions {
+  includeCount?: boolean;
+  includePayoutStatus?: boolean;
+}
+
 @Injectable()
 export class WinningsRepository {
   private readonly logger = new Logger(WinningsRepository.name);
@@ -162,8 +167,11 @@ export class WinningsRepository {
    */
   async searchWinnings(
     searchProps: WinningRequestDto,
+    options: SearchWinningsOptions = {},
   ): Promise<ResponseDto<SearchWinningResult>> {
     const result = new ResponseDto<SearchWinningResult>();
+    const includeCount = options.includeCount ?? true;
+    const includePayoutStatus = options.includePayoutStatus ?? true;
 
     try {
       let winnerIds: string[] | undefined;
@@ -202,7 +210,9 @@ export class WinningsRepository {
         !winnerIds && !!externalIds?.length,
       );
 
-      const winnings = await this.prisma.winnings.findMany({
+      const limit = searchProps.limit ?? 10;
+      const offset = searchProps.offset ?? 0;
+      const winningsPromise = this.prisma.winnings.findMany({
         where: queryWhere,
         include: {
           payment: {
@@ -218,15 +228,22 @@ export class WinningsRepository {
           origin: true,
         },
         orderBy,
-        skip: searchProps.offset,
-        take: searchProps.limit,
+        skip: offset,
+        take: limit,
       });
 
-      const count = await this.prisma.winnings.count({ where: queryWhere });
+      const [winnings, count] = includeCount
+        ? await Promise.all([
+            winningsPromise,
+            this.prisma.winnings.count({ where: queryWhere }),
+          ])
+        : [await winningsPromise, 0];
 
-      const usersPayoutStatusMap = winnings?.length
-        ? await this.getUsersPayoutStatusForWinnings(winnings)
-        : ({} as { [key: string]: payment_status });
+      const usersPayoutStatusMap: Record<string, unknown> =
+        includePayoutStatus && winnings?.length
+          ? await this.getUsersPayoutStatusForWinnings(winnings)
+          : {};
+      const totalItems = includeCount ? count : winnings.length;
 
       result.data = {
         winnings: winnings.map((item) => ({
@@ -257,13 +274,15 @@ export class WinningsRepository {
             item.payment?.[0].updated_at ??
             undefined) as Date,
           releaseDate: item.payment?.[0]?.release_date as Date,
-          paymentStatus: usersPayoutStatusMap[item.winner_id],
+          paymentStatus: usersPayoutStatusMap[
+            item.winner_id
+          ] as WinningDto['paymentStatus'],
         })),
         pagination: {
-          totalItems: count,
-          totalPages: Math.ceil(count / searchProps.limit),
-          pageSize: searchProps.limit,
-          currentPage: Math.ceil(searchProps.offset / searchProps.limit) + 1,
+          totalItems,
+          totalPages: Math.ceil(totalItems / limit),
+          pageSize: limit,
+          currentPage: Math.ceil(offset / limit) + 1,
         },
       };
       // response.data = winnings as any