|
| 1 | +import { Job, Queue, RateLimiter } from '../src' |
| 2 | + |
| 3 | +// Define job data types |
| 4 | +interface EmailJob { |
| 5 | + to: string |
| 6 | + subject: string |
| 7 | + body: string |
| 8 | +} |
| 9 | + |
| 10 | +interface ReportJob { |
| 11 | + reportId: string |
| 12 | + userId: string |
| 13 | + format: 'pdf' | 'csv' | 'xlsx' |
| 14 | +} |
| 15 | + |
| 16 | +interface ImageJob { |
| 17 | + imageId: string |
| 18 | + operations: Array<{ |
| 19 | + type: 'resize' | 'crop' | 'filter' |
| 20 | + params: Record<string, any> |
| 21 | + }> |
| 22 | +} |
| 23 | + |
| 24 | +async function main() { |
| 25 | + // Create queues for different job types |
| 26 | + const emailQueue = new Queue<EmailJob>('emails', { |
| 27 | + verbose: true, |
| 28 | + logLevel: 'info', |
| 29 | + metrics: { enabled: true }, |
| 30 | + defaultJobOptions: { |
| 31 | + attempts: 3, |
| 32 | + backoff: { |
| 33 | + type: 'exponential', |
| 34 | + delay: 1000, |
| 35 | + }, |
| 36 | + }, |
| 37 | + }) |
| 38 | + |
| 39 | + const reportQueue = new Queue<ReportJob>('reports') |
| 40 | + const imageQueue = new Queue<ImageJob>('images') |
| 41 | + |
| 42 | + // Set up event listeners for the email queue |
| 43 | + emailQueue.events.on('jobCompleted', (jobId, result) => { |
| 44 | + console.log(`Email job ${jobId} completed with result:`, result) |
| 45 | + }) |
| 46 | + |
| 47 | + emailQueue.events.on('jobFailed', (jobId, error) => { |
| 48 | + console.log(`Email job ${jobId} failed with error:`, error.message) |
| 49 | + }) |
| 50 | + |
| 51 | + // Create a rate limiter for the email queue (5 emails per second) |
| 52 | + const emailRateLimiter = new RateLimiter(emailQueue, { |
| 53 | + max: 5, |
| 54 | + duration: 1000, |
| 55 | + }) |
| 56 | + |
| 57 | + // Add jobs with dependencies |
| 58 | + // First, add a report generation job |
| 59 | + const reportJob = await reportQueue.add({ |
| 60 | + reportId: 'report-123', |
| 61 | + userId: 'user-456', |
| 62 | + format: 'pdf', |
| 63 | + }, { |
| 64 | + attempts: 2, |
| 65 | + jobId: 'report-job-123', |
| 66 | + }) |
| 67 | + |
| 68 | + console.log(`Added report job ${reportJob.id}`) |
| 69 | + |
| 70 | + // Then, add an email job that depends on the report job |
| 71 | + const emailJobWithDep = await emailQueue.add({ |
| 72 | + to: 'user@example.com', |
| 73 | + subject: 'Your report is ready', |
| 74 | + body: 'Please find your report attached.', |
| 75 | + }, { |
| 76 | + dependsOn: reportJob.id, // This job won't process until the report job is done |
| 77 | + jobId: 'email-job-123', |
| 78 | + }) |
| 79 | + |
| 80 | + console.log(`Added email job ${emailJobWithDep.id} depending on report job ${reportJob.id}`) |
| 81 | + |
| 82 | + // Add some image processing jobs |
| 83 | + for (let i = 1; i <= 3; i++) { |
| 84 | + const imageJob = await imageQueue.add({ |
| 85 | + imageId: `image-${i}`, |
| 86 | + operations: [ |
| 87 | + { type: 'resize', params: { width: 800, height: 600 } }, |
| 88 | + { type: 'filter', params: { name: 'grayscale' } }, |
| 89 | + ], |
| 90 | + }, { |
| 91 | + priority: i, // Higher number = higher priority |
| 92 | + }) |
| 93 | + |
| 94 | + console.log(`Added image job ${imageJob.id} with priority ${i}`) |
| 95 | + } |
| 96 | + |
| 97 | + // Add some rate-limited email jobs |
| 98 | + for (let i = 1; i <= 10; i++) { |
| 99 | + // Check rate limiter before adding the job |
| 100 | + const { limited, remaining, resetIn } = await emailRateLimiter.check() |
| 101 | + |
| 102 | + if (!limited) { |
| 103 | + const job = await emailQueue.add({ |
| 104 | + to: `user${i}@example.com`, |
| 105 | + subject: `Test Email ${i}`, |
| 106 | + body: `This is test email #${i}`, |
| 107 | + }) |
| 108 | + |
| 109 | + console.log(`Added rate-limited email job ${job.id}. Remaining: ${remaining}`) |
| 110 | + } |
| 111 | + else { |
| 112 | + console.log(`Rate limit reached. Try again in ${resetIn}ms. Skipping email ${i}.`) |
| 113 | + // In a real app, you might wait and retry, or queue locally |
| 114 | + await new Promise(resolve => setTimeout(resolve, 500)) |
| 115 | + i-- // Try again |
| 116 | + } |
| 117 | + } |
| 118 | + |
| 119 | + // Process report jobs |
| 120 | + reportQueue.process(2, async (job) => { |
| 121 | + console.log(`Processing report job ${job.id} for report ${job.data.reportId}`) |
| 122 | + |
| 123 | + // Update progress |
| 124 | + await job.updateProgress(25) |
| 125 | + |
| 126 | + // Simulate report generation work |
| 127 | + await new Promise(resolve => setTimeout(resolve, 2000)) |
| 128 | + |
| 129 | + await job.updateProgress(75) |
| 130 | + |
| 131 | + // More processing |
| 132 | + await new Promise(resolve => setTimeout(resolve, 1000)) |
| 133 | + |
| 134 | + await job.updateProgress(100) |
| 135 | + console.log(`Report ${job.data.reportId} generated successfully`) |
| 136 | + |
| 137 | + return { reportUrl: `https://example.com/reports/${job.data.reportId}.${job.data.format}` } |
| 138 | + }) |
| 139 | + |
| 140 | + // Process email jobs |
| 141 | + emailQueue.process(5, async (job) => { |
| 142 | + console.log(`Processing email job ${job.id} to ${job.data.to}`) |
| 143 | + |
| 144 | + // Update progress |
| 145 | + await job.updateProgress(50) |
| 146 | + |
| 147 | + // Simulate sending email |
| 148 | + await new Promise(resolve => setTimeout(resolve, 1000)) |
| 149 | + |
| 150 | + // Randomly fail some jobs to demonstrate retry |
| 151 | + if (Math.random() < 0.3) { |
| 152 | + throw new Error('Failed to send email: SMTP error') |
| 153 | + } |
| 154 | + |
| 155 | + await job.updateProgress(100) |
| 156 | + console.log(`Email sent successfully to ${job.data.to}`) |
| 157 | + |
| 158 | + return { |
| 159 | + sent: true, |
| 160 | + messageId: `msg-${Date.now()}`, |
| 161 | + timestamp: new Date().toISOString(), |
| 162 | + } |
| 163 | + }) |
| 164 | + |
| 165 | + // Process image jobs |
| 166 | + imageQueue.process(3, async (job) => { |
| 167 | + console.log(`Processing image job ${job.id} for image ${job.data.imageId}`) |
| 168 | + |
| 169 | + // Update progress |
| 170 | + await job.updateProgress(25) |
| 171 | + |
| 172 | + // For each operation |
| 173 | + for (const [index, operation] of job.data.operations.entries()) { |
| 174 | + console.log(`Applying ${operation.type} to image ${job.data.imageId}`) |
| 175 | + |
| 176 | + // Simulate image processing work |
| 177 | + await new Promise(resolve => setTimeout(resolve, 1000)) |
| 178 | + |
| 179 | + const progressIncrement = 75 / job.data.operations.length |
| 180 | + await job.updateProgress(25 + progressIncrement * (index + 1)) |
| 181 | + } |
| 182 | + |
| 183 | + await job.updateProgress(100) |
| 184 | + console.log(`Image ${job.data.imageId} processed successfully`) |
| 185 | + |
| 186 | + return { |
| 187 | + imageUrl: `https://example.com/images/${job.data.imageId}-processed.jpg`, |
| 188 | + operations: job.data.operations.length, |
| 189 | + } |
| 190 | + }) |
| 191 | + |
| 192 | + // Display job counts and metrics periodically |
| 193 | + const interval = setInterval(async () => { |
| 194 | + // Get counts for all queues |
| 195 | + const [emailCounts, reportCounts, imageCounts] = await Promise.all([ |
| 196 | + emailQueue.getJobCounts(), |
| 197 | + reportQueue.getJobCounts(), |
| 198 | + imageQueue.getJobCounts(), |
| 199 | + ]) |
| 200 | + |
| 201 | + console.log('\nJob Counts:') |
| 202 | + console.log('Email Queue:', emailCounts) |
| 203 | + console.log('Report Queue:', reportCounts) |
| 204 | + console.log('Image Queue:', imageCounts) |
| 205 | + |
| 206 | + // Get metrics |
| 207 | + const metrics = await emailQueue.getMetrics() |
| 208 | + if (metrics) { |
| 209 | + console.log('\nEmail Queue Metrics:') |
| 210 | + console.log(`Processed per minute: ${metrics.jobsProcessedPerMinute}`) |
| 211 | + console.log(`Added per minute: ${metrics.jobsAddedPerMinute}`) |
| 212 | + console.log(`Error rate: ${(metrics.errorRate * 100).toFixed(1)}%`) |
| 213 | + console.log(`Avg processing time: ${metrics.processingTime.avg.toFixed(0)}ms`) |
| 214 | + } |
| 215 | + |
| 216 | + // Check if all jobs are completed or failed |
| 217 | + const totalActive = emailCounts.active + reportCounts.active + imageCounts.active |
| 218 | + const totalWaiting = emailCounts.waiting + reportCounts.waiting + imageCounts.waiting |
| 219 | + const totalDelayed = emailCounts.delayed + reportCounts.delayed + imageCounts.delayed |
| 220 | + |
| 221 | + if (totalActive === 0 && totalWaiting === 0 && totalDelayed === 0) { |
| 222 | + console.log('\nAll jobs processed!') |
| 223 | + |
| 224 | + // Display final results |
| 225 | + const completed = emailCounts.completed + reportCounts.completed + imageCounts.completed |
| 226 | + const failed = emailCounts.failed + reportCounts.failed + imageCounts.failed |
| 227 | + console.log(`Total completed: ${completed}`) |
| 228 | + console.log(`Total failed: ${failed}`) |
| 229 | + |
| 230 | + // Clean up |
| 231 | + clearInterval(interval) |
| 232 | + |
| 233 | + // Close all queues |
| 234 | + await Promise.all([ |
| 235 | + emailQueue.close(), |
| 236 | + reportQueue.close(), |
| 237 | + imageQueue.close(), |
| 238 | + ]) |
| 239 | + |
| 240 | + console.log('All queues closed, exiting...') |
| 241 | + } |
| 242 | + }, 2000) |
| 243 | +} |
| 244 | + |
| 245 | +main().catch(console.error) |
0 commit comments