feat: enhance queue system with new SQLite and Redis adapters, update dependencies, and improve project structure

Author: muniter

example-queue-project/README.md

@@ -13,7 +13,8 @@ This example shows the **separation of concerns** pattern recommended for produc
 
 ## Features Demonstrated
 
-- **Multiple Queue Types**: Database queue (SQLite) and File queue
+- **Multiple Queue Types**: SQLite, Redis, SQS, and File queues with clean adapter APIs
+- **New Adapter System**: Clean drizzle-style APIs for queue creation
 - **Type Safety**: Full TypeScript support with job payload validation
 - **Event System**: Comprehensive logging via queue lifecycle events
 - **Worker Management**: Using Worker class for clean process organization
@@ -28,7 +29,24 @@ This example shows the **separation of concerns** pattern recommended for produc
 pnpm install
 ```
 
-2. Build the project:
+2. Install the queue drivers you want to use (peer dependencies):
+
+For SQLite queue:
+```bash
+pnpm add better-sqlite3
+```
+
+For Redis queue:
+```bash
+pnpm add redis
+```
+
+For SQS queue:
+```bash
+pnpm add @aws-sdk/client-sqs
+```
+
+3. Build the project:
 ```bash
 pnpm run build
 ```
@@ -88,13 +106,10 @@ In a real production environment, you would:
 
 ## Files
 
-- `queues.ts` - Centralized queue configuration, initialization, and event listeners
-- `add-job.ts` - Job producer (adds jobs to queues)
-- `process-jobs.ts` - Job consumer (processes jobs continuously)  
-- `types.ts` - Shared job type definitions
-- `sqlite-adapter.ts` - Database adapter implementation
-- `database.ts` - Database initialization
-- `index.ts` - Original combined example (legacy)
+- `general-queue.ts` - SQLite queue using new adapter API
+- `redis-queue.ts` - Redis queue using new adapter API  
+- `email-queue.ts` - File and SQS queues using new adapter APIs
+- `index.ts` - Main demo entry point
 
 ## Try It
 

example-queue-project/package.json

@@ -18,15 +18,16 @@
     "redis:logs": "docker-compose logs -f redis"
   },
   "dependencies": {
+    "@muniter/queue": "link:..",
+    "dotenv": "^16.3.1"
+  },
+  "peerDependencies": {
     "@aws-sdk/client-sqs": "^3.831.0",
-    "@muniter/queue": "file:..",
-    "dotenv": "^16.3.1",
-    "redis": "^4.6.0",
-    "sqlite3": "^5.1.6"
+    "better-sqlite3": "^12.0.0",
+    "redis": "^5.5.6"
   },
   "devDependencies": {
     "@types/node": "^20.10.0",
-    "@types/sqlite3": "^3.1.11",
     "tsx": "^4.6.2",
     "typescript": "^5.3.3"
   }

example-queue-project/src/database.ts

@@ -1,5 +1,5 @@
-import { SQS } from "@aws-sdk/client-sqs";
-import { FileQueue, SqsQueue } from "@muniter/queue";
+import { FileQueue } from "@muniter/queue";
+import { createSQSQueue } from "@muniter/queue/sqs";
 
 interface EmailJobs {
   "welcome-email": { to: string; name: string };
@@ -11,15 +11,12 @@ export const emailQueueFile = new FileQueue<EmailJobs>({
   path: "./email-queue",
 });
 
-// SQS-based queue (for production)
-const sqsClient = new SQS({
-  region: "us-east-1",
-  profile: "javier",
-});
-
-export const emailQueueSqs = new SqsQueue<EmailJobs>(
-  sqsClient,
-  "https://sqs.us-east-1.amazonaws.com/428011609647/test-queue"
+// SQS-based queue (for production) - simple API
+export const emailQueueSqs = createSQSQueue<EmailJobs>(
+  "https://sqs.us-east-1.amazonaws.com/428011609647/test-queue",
+  {
+    profile: "javier",
+  }
 );
 
 export const emailQueue = emailQueueSqs;
@@ -52,25 +49,29 @@ emailQueue.onJob("notification", async (payload) => {
 
 emailQueue.on("beforeExec", (event) => {
   console.log(
-    `\n[emailQueue][${emailQueue.constructor.name}][${new Date().toISOString()}] Starting ${event.name} job ${
-      event.id
-    }...`
+    `\n[emailQueue][${
+      emailQueue.constructor.name
+    }][${new Date().toISOString()}] Starting ${event.name} job ${event.id}...`
   );
 });
 
 emailQueue.on("afterExec", (event) => {
   console.log(
-    `[emailQueue][${emailQueue.constructor.name}][${new Date().toISOString()}] Email job ${event.id} (${
+    `[emailQueue][${
+      emailQueue.constructor.name
+    }][${new Date().toISOString()}] Email job ${event.id} (${
       event.name
     }) completed successfully`
   );
 });
 
 emailQueue.on("afterError", (event) => {
   console.error(
-    `[emailQueue][${emailQueue.constructor.name}][${new Date().toISOString()}] Email job ${event.id} (${
+    `[emailQueue][${
+      emailQueue.constructor.name
+    }][${new Date().toISOString()}] Email job ${event.id} (${
       event.name
     }) failed:`,
     event.error
   );
-});
+});

example-queue-project/src/email-queue.ts

@@ -1,13 +1,11 @@
-import { DbQueue } from "@muniter/queue";
-import { SQLiteDatabaseAdapter } from "./sqlite-adapter.js";
+import { createSQLiteQueue } from "@muniter/queue/sqlite";
 
 export interface GeneralJobs {
     'process-image': { url: string; width: number; height: number };
     'generate-report': { type: string; period: string };
 }
 
-
-export const generalQueue = new DbQueue<GeneralJobs>(new SQLiteDatabaseAdapter());
+export const generalQueue = createSQLiteQueue<GeneralJobs>('queue.db');
 
 // Register job handlers for general queue
 generalQueue.onJob('process-image', async (payload) => {

example-queue-project/src/general-queue.ts

@@ -1,41 +1,43 @@
-import { initializeDatabase } from './database.js';
 import { emailQueue } from './email-queue.js';
 import { generalQueue } from './general-queue.js';
+import { emailQueue as redisEmailQueue } from './redis-queue.js';
 import { parseArgs } from 'util';
 
 async function push() {
-  console.log('Initializing database...');
-  await initializeDatabase();
+  console.log('Adding jobs to demonstrate different queue types...');
+  console.log('- SQLite queue (general tasks)');
+  console.log('- SQS queue (production email)'); 
+  console.log('- Redis queue (high-performance email)');
+  console.log('');
 
-  console.log('Adding some test jobs...');
-  // Add jobs using the new type-safe API
+  // SQS queue - using new createQueue(queueUrl) API
   const emailJobId1 = await emailQueue.addJob('welcome-email', {
     payload: {
       to: 'user@example.com',
       name: 'John Doe'
     }
   });
-  console.log(`Welcome email job added with ID: ${emailJobId1}`);
+  console.log(`[SQS] Welcome email job added with ID: ${emailJobId1}`);
 
-  const emailJobId2 = await emailQueue.addJob('notification', {
+  // Redis queue - using new createRedisQueue(url) API  
+  const emailJobId2 = await redisEmailQueue.addJob('notification', {
     payload: {
       to: 'admin@example.com',
       subject: 'Daily Report',
       body: 'Here is your daily report...'
     }
-    // Note: FileQueue doesn't support priority - would cause TypeScript error
-    // For demo purposes, we removed the priority option
   });
-  console.log(`Priority notification job added with ID: ${emailJobId2}`);
+  console.log(`[Redis] Notification job added with ID: ${emailJobId2}`);
 
+  // SQLite queue - using new createQueue('db.sqlite') API
   const imageJobId = await generalQueue.addJob('process-image', {
     payload: {
       url: 'https://example.com/image.jpg',
       width: 800,
       height: 600
     }
   });
-  console.log(`Image job added with ID: ${imageJobId}`);
+  console.log(`[SQLite] Image job added with ID: ${imageJobId}`);
 
   await generalQueue.addJob('generate-report', {
     payload: {
@@ -44,6 +46,7 @@ async function push() {
     },
     delay: 2
   });
+  console.log(`[SQLite] Report job added with 2s delay`);
 }
 
 async function run() {

example-queue-project/src/index.ts

@@ -1,22 +1,12 @@
-import { RedisQueue } from "@muniter/queue";
-import { createClient } from "redis";
+import { createRedisQueue } from "@muniter/queue/redis";
 
 interface EmailJobs {
   "welcome-email": { to: string; name: string };
   "notification": { to: string; subject: string; body: string };
 }
 
-// Create Redis client directly
-const redisClient = createClient({ url: 'redis://localhost:6379' });
-
-redisClient.on('error', (err) => console.error('Redis Client Error:', err));
-redisClient.on('connect', () => console.log('Redis Client Connected'));
-redisClient.on('ready', () => console.log('Redis Client Ready'));
-
-await redisClient.connect();
-
-// Create Redis queue - no adapter needed!
-export const emailQueue = new RedisQueue<EmailJobs>(redisClient, 'email');
+// Create Redis queue with simple API
+export const emailQueue = createRedisQueue<EmailJobs>('redis://localhost:6379');
 
 // Register job handlers
 emailQueue.onJob("welcome-email", async (payload) => {