Basic email open tracking

Author: Rajat Saxena

apps/queue/.env

@@ -6,4 +6,5 @@ DB_CONNECTION_STRING=mongodb://db.string
 REDIS_HOST=localhost
 REDIS_PORT=6379
 SEQUENCE_BOUNCE_LIMIT=3
-DOMAIN=courselit.app
+DOMAIN=courselit.app
+PIXEL_SIGNING_SECRET=super_secret_string

apps/queue/src/domain/model/email-delivery.ts

@@ -0,0 +1,5 @@
+import mongoose from "mongoose";
+import { EmailDeliverySchema } from "@courselit/common-logic";
+
+export default mongoose.models.EmailDelivery ||
+    mongoose.model("EmailDelivery", EmailDeliverySchema);

apps/queue/src/domain/process-ongoing-sequences.ts

@@ -19,9 +19,12 @@ import { Worker } from "bullmq";
 import redis from "../redis";
 import mongoose from "mongoose";
 import sequenceQueue from "./sequence-queue";
+import EmailDelivery from "./model/email-delivery";
 import { AdminSequence, InternalUser } from "@courselit/common-logic";
-import { renderEmailToHtml } from "@courselit/email-editor";
+import { Email as EmailType, renderEmailToHtml } from "@courselit/email-editor";
 import { getUnsubLink } from "../utils/get-unsub-link";
+import { getSiteUrl } from "../utils/get-site-url";
+import { jwtUtils } from "@courselit/utils";
 const liquidEngine = new Liquid();
 
 new Worker(
@@ -38,6 +41,11 @@ new Worker(
 );
 
 export async function processOngoingSequences(): Promise<void> {
+    if (!process.env.PIXEL_SIGNING_SECRET) {
+        throw new Error(
+            "PIXEL_SIGNING_SECRET environment variable is not defined",
+        );
+    }
     // eslint-disable-next-line no-constant-condition
     while (true) {
         // eslint-disable-next-line no-console
@@ -194,9 +202,37 @@ async function attemptMailSending({
         return;
     }
     // const content = email.content;
+    const pixelPayload = {
+        userId: user.userId,
+        sequenceId: ongoingSequence.sequenceId,
+        emailId: email.emailId,
+    };
+    const pixelToken = jwtUtils.generateToken(
+        pixelPayload,
+        process.env.PIXEL_SIGNING_SECRET,
+        "365d",
+    );
+    const pixelUrl = `${getSiteUrl(domain)}/api/pixel?d=${pixelToken}`;
+    const emailContentWithPixel: EmailType = {
+        content: [
+            ...email.content.content,
+            {
+                blockType: "image",
+                settings: {
+                    src: pixelUrl,
+                    width: "1px",
+                    height: "1px",
+                    alt: "CourseLit Pixel",
+                },
+            },
+        ],
+        style: email.content.style,
+        meta: email.content.meta,
+    };
+
     const content = await liquidEngine.parseAndRender(
         await renderEmailToHtml({
-            email: email.content,
+            email: emailContentWithPixel,
         }),
         templatePayload,
     );
@@ -207,6 +243,13 @@ async function attemptMailSending({
             subject,
             html: content,
         });
+        // @ts-ignore - Mongoose type compatibility issue
+        await EmailDelivery.create({
+            domain: (domain as any).id,
+            sequenceId: sequence.sequenceId,
+            userId: user.userId,
+            emailId: email.emailId,
+        });
     } catch (err: any) {
         ongoingSequence.retryCount++;
         if (ongoingSequence.retryCount >= sequenceBounceLimit) {

apps/web/app/api/pixel/route.ts

@@ -0,0 +1,90 @@
+import { NextRequest, NextResponse } from "next/server";
+import DomainModel, { Domain } from "@models/Domain";
+import EmailEventModel from "@models/EmailEvent";
+import UserModel from "@models/User";
+import SequenceModel from "@models/Sequence";
+import { Constants, Sequence, User } from "@courselit/common-models";
+import { error } from "@/services/logger";
+import { jwtUtils } from "@courselit/utils";
+
+function getJwtSecret(): string {
+    const jwtSecret = process.env.PIXEL_SIGNING_SECRET;
+    if (!jwtSecret) {
+        throw new Error("PIXEL_SIGNING_SECRET is not defined");
+    }
+    return jwtSecret;
+}
+
+// 1x1 transparent PNG buffer
+const pixelBuffer = Buffer.from(
+    "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/w8AAgMBApUe1ZkAAAAASUVORK5CYII=",
+    "base64",
+);
+
+const pixelResponse = new NextResponse(pixelBuffer, {
+    status: 200,
+    headers: {
+        "Content-Type": "image/png",
+        "Cache-Control": "no-store",
+    },
+});
+
+export async function GET(req: NextRequest) {
+    if (!process.env.PIXEL_SIGNING_SECRET) {
+        error(
+            "PIXEL_SIGNING_SECRET environment variable is not defined. No pixel tracking is done.",
+        );
+        return pixelResponse;
+    }
+
+    try {
+        const domainName = req.headers.get("domain");
+        const domain = await DomainModel.findOne<Domain>({
+            name: domainName,
+        });
+        if (!domain) {
+            throw new Error(`Domain not found: ${domainName}`);
+        }
+
+        const { searchParams } = new URL(req.url);
+        const d = searchParams.get("d");
+        if (!d) {
+            throw new Error("Missing data query parameter");
+        }
+
+        const jwtSecret = getJwtSecret();
+        const payload = jwtUtils.verifyToken(d, jwtSecret);
+        const { userId, sequenceId, emailId } = payload as any;
+        if (!userId || !sequenceId || !emailId) {
+            throw new Error(
+                `Invalid payload: Not all required fields are present: ${JSON.stringify(payload)}`,
+            );
+        }
+
+        const sequence = await SequenceModel.findOne<Sequence>({
+            domain: domain._id,
+            sequenceId,
+        });
+        const user = await UserModel.findOne<User>({
+            domain: domain._id,
+            userId,
+        });
+        const email = sequence?.emails.find((e) => e.emailId === emailId);
+        if (sequence && user && email) {
+            await EmailEventModel.create({
+                domain: domain._id,
+                sequenceId,
+                userId,
+                emailId,
+                action: Constants.EmailEventAction.OPEN,
+            });
+        }
+    } catch (err) {
+        error(`Invalid pixel data`, {
+            fileName: "pixel.route.ts",
+            stack: err.stack,
+        });
+    }
+
+    return pixelResponse;
+}

apps/web/models/EmailEvent.ts

@@ -0,0 +1,5 @@
+import mongoose from "mongoose";
+import { EmailEventSchema } from "@courselit/common-logic";
+
+export default mongoose.models.EmailEvent ||
+    mongoose.model("EmailEvent", EmailEventSchema);

packages/common-logic/src/index.ts

@@ -8,3 +8,5 @@ export * from "./models/user-filter";
 export * from "./models/course";
 export * from "./models/rule";
 export * from "./models/email";
+export * from "./models/email-delivery";
+export * from "./models/email-event";

packages/common-logic/src/models/email-delivery.ts

@@ -0,0 +1,13 @@
+import mongoose from "mongoose";
+
+export const EmailDeliverySchema = new mongoose.Schema(
+    {
+        domain: { type: mongoose.Schema.Types.ObjectId, required: true },
+        sequenceId: { type: String, required: true },
+        userId: { type: String, required: true },
+        emailId: { type: String, required: true },
+    },
+    {
+        timestamps: { createdAt: true, updatedAt: false },
+    },
+);

packages/common-logic/src/models/email-event.ts

@@ -0,0 +1,22 @@
+import mongoose from "mongoose";
+import { Constants } from "@courselit/common-models";
+
+export const EmailEventSchema = new mongoose.Schema(
+    {
+        domain: { type: mongoose.Schema.Types.ObjectId, required: true },
+        sequenceId: { type: String, required: true },
+        userId: { type: String, required: true },
+        emailId: { type: String, required: true },
+        action: {
+            type: String,
+            required: true,
+            enum: Object.values(Constants.EmailEventAction),
+        },
+        linkId: { type: String },
+        bounceType: { type: String, enum: ["hard", "soft"] },
+        bounceReason: { type: String },
+    },
+    {
+        timestamps: true,
+    },
+);

packages/common-models/src/constants.ts

@@ -176,3 +176,8 @@ export const EventType = {
     COMMUNITY_JOINED: "community:joined",
     COMMUNITY_LEFT: "community:left",
 } as const;
+export const EmailEventAction = {
+    OPEN: "open",
+    CLICK: "click",
+    BOUNCE: "bounce",
+} as const;

packages/common-models/src/email-event-action.ts

@@ -0,0 +1,4 @@
+import { Constants } from ".";
+
+export type EmailEventAction =
+    (typeof Constants.EmailEventAction)[keyof typeof Constants.EmailEventAction];