feat: refactor expired trash items job to bullmq

Author: apsantiso

.env.template

@@ -24,6 +24,7 @@ RDS_USERNAME=postgres
 RECAPTCHA_V3=captcha_secret
 RECAPTCHA_V3_ENDPOINT=https://www.google.com/recaptcha/api/siteverify
 REDIS_CONNECTION_STRING=redis://@drive-cache:6380
+
 SENDGRID_API_KEY=sendgrid_api_key
 SENDGRID_MODE_SANDBOX=true
 SENDGRID_TEMPLATE_DRIVE_WELCOME_EMAIL_VERIFICATION=sendgrid_template_id
@@ -71,3 +72,6 @@ APN_BUNDLE_ID="APN_BUNDLE_ID"
 APN_KEY_ID=APN_KEY_ID
 APN_ENCRYPTION_KEY=APN_ENCRYPTION_KEY
 APN_SECRET=APN_JWT_SECRET
+
+## Not required if EXECUTE_JOBS != true
+REDIS_JOBS_CONNECTION_STRING=redis://@drive-cache:6380

package.json

@@ -40,6 +40,7 @@
     "@keyv/redis": "^5.1.6",
     "@nest-lab/throttler-storage-redis": "^1.2.0",
     "@nestjs/axios": "^4.0.1",
+    "@nestjs/bullmq": "^11.0.4",
     "@nestjs/cache-manager": "^3.1.0",
     "@nestjs/cli": "^11.0.16",
     "@nestjs/common": "^11.1.14",
@@ -59,6 +60,7 @@
     "ajv": "^8.18.0",
     "axios": "^1.13.6",
     "bcryptjs": "^2.4.3",
+    "bullmq": "^5.71.0",
     "cache-manager": "^6.4.3",
     "class-transformer": "^0.5.1",
     "class-validator": "^0.15.1",

src/app.module.ts

@@ -33,6 +33,8 @@ import { AuthGuard } from './modules/auth/auth.guard';
 import { CacheManagerModule } from './modules/cache-manager/cache-manager.module';
 import { ReferralModule } from './modules/referral/referral.module';
 
+const isCronjobInstance = process.env.EXECUTE_JOBS === 'true';
+
 @Module({
   imports: [
     LoggerModule.forRoot({
@@ -124,7 +126,7 @@ import { ReferralModule } from './modules/referral/referral.module';
       }),
     }),
     EventEmitterModule.forRoot({ wildcard: true, delimiter: '.' }),
-    JobsModule,
+    ...(isCronjobInstance ? [JobsModule] : []),
     NotificationModule,
     NotificationsModule,
     FileModule,

src/config/configuration.ts

@@ -34,6 +34,7 @@ export default () => ({
   },
   cache: {
     redisConnectionString: process.env.REDIS_CONNECTION_STRING,
+    redisJobsConnection: process.env.REDIS_JOBS_CONNECTION_STRING,
   },
   secrets: {
     cryptoSecret: process.env.CRYPTO_SECRET,

src/modules/jobs/jobs.module.ts

@@ -1,6 +1,8 @@
 import { Module } from '@nestjs/common';
 import { SequelizeModule } from '@nestjs/sequelize';
 import { ScheduleModule } from '@nestjs/schedule';
+import { BullModule } from '@nestjs/bullmq';
+import { ConfigService } from '@nestjs/config';
 import { DeletedItemsCleanupTask } from './tasks/deleted-items-cleanup.task';
 import { FileModule } from '../file/file.module';
 import { FolderModule } from '../folder/folder.module';
@@ -14,12 +16,33 @@ import { MailerModule } from '../../externals/mailer/mailer.module';
 import { FeatureLimitModule } from '../feature-limit/feature-limit.module';
 import { SecurityModule } from '../security/security.module';
 import { DeleteExpiredFileVersionsTask } from './tasks/delete-expired-file-versions.task';
-import { DeleteExpiredTrashItemsTask } from './tasks/delete-expired-trash-items.task';
-
+import { TRASH_CLEANUP_QUEUE } from './tasks/trash-cleanup/trash-cleanup.scheduler';
+import { TrashCleanupScheduler } from './tasks/trash-cleanup/trash-cleanup.scheduler';
+import { TrashCleanupProcessor } from './tasks/trash-cleanup/trash-cleanup.processor';
 @Module({
   imports: [
     SequelizeModule.forFeature([JobExecutionModel]),
     ScheduleModule.forRoot(),
+    BullModule.forRootAsync({
+      inject: [ConfigService],
+      useFactory: (configService: ConfigService) => {
+        const url = new URL(
+          configService.get<string>('cache.redisJobsConnection'),
+        );
+        return {
+          connection: {
+            host: url.hostname,
+            port: Number(url.port) || 6379,
+            password: url.password || undefined,
+            username: url.username || undefined,
+            maxRetriesPerRequest: null,
+            enableOfflineQueue: false,
+            tls: {},
+          },
+        };
+      },
+    }),
+    BullModule.registerQueue({ name: TRASH_CLEANUP_QUEUE }),
     FileModule,
     FolderModule,
     UserModule,
@@ -34,7 +57,8 @@ import { DeleteExpiredTrashItemsTask } from './tasks/delete-expired-trash-items.
     RetroActiveDeleteItemsCleanupTask,
     InactiveUsersEmailTask,
     DeleteExpiredFileVersionsTask,
-    DeleteExpiredTrashItemsTask,
+    TrashCleanupScheduler,
+    TrashCleanupProcessor,
   ],
 })
 export class JobsModule {}