Skip to content

Commit de7ac01

Browse files
malek10xdevmalek10xdev
committed
fix(email-campaigns): batch large contact lookups to avoid URI too long
Large email campaigns (>~1k contacts) failed with "URI too long" because both `getSelectedContacts` (compliance middleware) and `getContactsByEmails` (/campaigns/preview) used a single `\.in("email", emails)` query. PostgREST serializes that as `?email=in.(...)` in the URL, and ~30 chars/email blows past the URL length limit on Supabase / edge functions. Batch the lookup into chunks of 100, re-sort the merged rows so the "most-recently-updated wins" dedup semantic is preserved across batches, and gate `Deno.serve()` behind `import.meta.main` so the module can be imported by tests without binding a port. The same batching pattern (size 100) already exists in leadminer-commercial's copy of middlewares-mod.ts; this PR brings parity to the open-source repo. Also fixes 6 pre-existing broken tests in tests/campaign-{bill,check}- middleware.test.ts that imported non-existent files (campaign-{bill,check}-middleware.ts). The real middlewares live in middlewares-mod.ts as `createFinalResponseMiddleware` and `complianceMiddleware`. New tests: - tests/get-selected-contacts.test.ts (4) - tests/get-contacts-by-emails.test.ts (3) All 22 edge-function tests pass.
1 parent 7be4884 commit de7ac01

6 files changed

Lines changed: 520 additions & 63 deletions

File tree

‎supabase/functions/email-campaigns/index.ts‎

Lines changed: 43 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -929,7 +929,7 @@ async function getSelectedContacts(
929929
return contacts;
930930
}
931931

932-
async function getContactsByEmails(
932+
export async function getContactsByEmails(
933933
supabaseAdmin: ReturnType<typeof createSupabaseAdmin>,
934934
userId: string,
935935
emails: string[],
@@ -941,20 +941,44 @@ async function getContactsByEmails(
941941
new Set(emails.map((e) => e.toLowerCase().trim()).filter(Boolean)),
942942
);
943943

944-
const { data: personRows, error: personErr } = await supabaseAdmin
945-
.schema("private")
946-
.from("persons")
947-
.select("id, email")
948-
.eq("user_id", userId)
949-
.in("email", normalizedEmails);
950-
951-
if (personErr) {
952-
throw new Error(
953-
`Unable to fetch contacts for campaign: ${personErr.message}`,
944+
if (normalizedEmails.length === 0) return [] as ContactSnapshot[];
945+
946+
// PostgREST serializes `.in("email", emails)` as `?email=in.(...)` in the URL.
947+
// At ~30 chars/email the request blows past PostgREST's URL length limit for
948+
// large campaigns, surfacing as "URI too long". Batch the email->id lookup
949+
// to keep each query string under the limit.
950+
const GET_CONTACTS_BY_EMAILS_BATCH_SIZE = 100;
951+
const allPersonRows: Array<{ id: string; email: string }> = [];
952+
for (
953+
let i = 0;
954+
i < normalizedEmails.length;
955+
i += GET_CONTACTS_BY_EMAILS_BATCH_SIZE
956+
) {
957+
const chunk = normalizedEmails.slice(
958+
i,
959+
i + GET_CONTACTS_BY_EMAILS_BATCH_SIZE,
954960
);
961+
const { data: personRows, error: personErr } = await supabaseAdmin
962+
.schema("private")
963+
.from("persons")
964+
.select("id, email")
965+
.eq("user_id", userId)
966+
.in("email", chunk);
967+
968+
if (personErr) {
969+
throw new Error(
970+
`Unable to fetch contacts for campaign: ${personErr.message}`,
971+
);
972+
}
973+
974+
if (personRows) {
975+
allPersonRows.push(
976+
...(personRows as Array<{ id: string; email: string }>),
977+
);
978+
}
955979
}
956980

957-
const ids = (personRows ?? []).map((r) => r.id as string);
981+
const ids = allPersonRows.map((r) => r.id);
958982
if (!ids.length) return [] as ContactSnapshot[];
959983

960984
const { data, error } = await supabaseAdmin
@@ -2585,4 +2609,10 @@ async function sendOAuthFailureNotification(
25852609
}
25862610
}
25872611

2588-
Deno.serve((req) => app.fetch(req));
2612+
// Guard: only start the HTTP server when this module is the entrypoint.
2613+
// `import.meta.main` is `true` for `deno run index.ts` (production / supabase
2614+
// functions serve) and `false` for `import("./index.ts")` in tests, where we
2615+
// want to import helpers like `getContactsByEmails` without binding a port.
2616+
if (import.meta.main) {
2617+
Deno.serve((req) => app.fetch(req));
2618+
}

‎supabase/functions/email-campaigns/middlewares-mod.ts‎

Lines changed: 57 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { Context, Next } from "hono";
22
import { createSupabaseAdmin } from "../_shared/supabase.ts";
33
import { createLogger } from "../_shared/logger.ts";
4-
import { initI18n, t, getUserLocale } from "./i18n.ts";
4+
import { getUserLocale, initI18n, t } from "./i18n.ts";
55

66
// Constants
77
const PRIVACY_POLICY_URL = "/privacy-policy";
@@ -50,32 +50,72 @@ interface CheckResult {
5050
eligibleCount?: number;
5151
}
5252

53-
async function getSelectedContacts(
53+
// PostgREST serializes `.in("email", emails)` as `?email=in.(...)` in the URL.
54+
// At ~30 chars/email the request blows past PostgREST's URL length limit for
55+
// large campaigns, surfacing as "URI too long". Batch the lookup to keep each
56+
// query string under the limit while preserving the original dedup semantic.
57+
const SELECTED_CONTACTS_BATCH_SIZE = 100;
58+
59+
export async function getSelectedContacts(
5460
supabaseAdmin: ReturnType<typeof createSupabaseAdmin>,
5561
userId: string,
5662
emails: string[],
5763
): Promise<ContactSnapshot[]> {
58-
const { data, error } = await supabaseAdmin
59-
.schema("private")
60-
.from("persons")
61-
.select("email, consent_status, updated_at")
62-
.eq("user_id", userId)
63-
.in("email", emails)
64-
.order("updated_at", { ascending: false });
65-
66-
if (error) {
67-
throw new Error(`Failed to fetch contacts: ${error.message}`);
64+
if (emails.length === 0) {
65+
return [];
66+
}
67+
68+
const allRows: Array<{
69+
email: string;
70+
consent_status: string;
71+
updated_at: string;
72+
}> = [];
73+
74+
for (let i = 0; i < emails.length; i += SELECTED_CONTACTS_BATCH_SIZE) {
75+
const chunk = emails.slice(i, i + SELECTED_CONTACTS_BATCH_SIZE);
76+
const { data, error } = await supabaseAdmin
77+
.schema("private")
78+
.from("persons")
79+
.select("email, consent_status, updated_at")
80+
.eq("user_id", userId)
81+
.in("email", chunk)
82+
.order("updated_at", { ascending: false });
83+
84+
if (error) {
85+
throw new Error(`Failed to fetch contacts: ${error.message}`);
86+
}
87+
88+
if (data) {
89+
allRows.push(
90+
...(data as Array<{
91+
email: string;
92+
consent_status: string;
93+
updated_at: string;
94+
}>),
95+
);
96+
}
6897
}
6998

99+
// Re-sort the merged rows so the "most-recently-updated wins" dedup semantic
100+
// (originally provided by `.order("updated_at", { ascending: false })` on a
101+
// single query) still holds when rows come from multiple batched queries.
102+
allRows.sort(
103+
(a, b) =>
104+
new Date(b.updated_at).getTime() - new Date(a.updated_at).getTime(),
105+
);
106+
70107
const contactsByEmail = new Map<string, ContactSnapshot>();
71108

72-
for (const row of data || []) {
109+
for (const row of allRows) {
73110
const key = row.email.toLowerCase();
74111
if (contactsByEmail.has(key)) continue;
75112

76113
contactsByEmail.set(key, {
77114
email: row.email,
78-
consent_status: row.consent_status || "legitimate_interest",
115+
consent_status:
116+
(row.consent_status || "legitimate_interest") as ContactSnapshot[
117+
"consent_status"
118+
],
79119
updated_at: row.updated_at,
80120
});
81121
}
@@ -258,8 +298,9 @@ export async function complianceMiddleware(c: Context, next: Next) {
258298
);
259299

260300
if (complianceCheck.response) {
261-
const statusCode =
262-
complianceCheck.response.data?.available === 0 ? 400 : 266;
301+
const statusCode = complianceCheck.response.data?.available === 0
302+
? 400
303+
: 266;
263304

264305
// Add partial_continue field if applicable
265306
if (complianceCheck.partialField) {

‎supabase/functions/email-campaigns/tests/campaign-bill-middleware.test.ts‎

Lines changed: 41 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
import { Context } from "hono";
22

33
Deno.test({
4-
name: "campaign-bill-middleware: should skip non-create paths",
4+
name: "createFinalResponseMiddleware: should skip non-create paths",
55
async fn() {
6-
const { campaignBillMiddleware } =
7-
await import("../campaign-bill-middleware.ts");
6+
const { createFinalResponseMiddleware } = await import(
7+
"../middlewares-mod.ts"
8+
);
89

910
let nextCalled = false;
1011
// skipcq: JS-0323 - Test mock requires minimal Context interface
@@ -15,7 +16,7 @@ Deno.test({
1516
json: () => {},
1617
} as unknown as Context;
1718

18-
await campaignBillMiddleware(context, async () => {
19+
await createFinalResponseMiddleware(context, async () => {
1920
nextCalled = true;
2021
});
2122

@@ -26,10 +27,12 @@ Deno.test({
2627
});
2728

2829
Deno.test({
29-
name: "campaign-bill-middleware: should return 500 when campaign data missing",
30+
name:
31+
"createFinalResponseMiddleware: should return 500 when campaign data missing",
3032
async fn() {
31-
const { campaignBillMiddleware } =
32-
await import("../campaign-bill-middleware.ts");
33+
const { createFinalResponseMiddleware } = await import(
34+
"../middlewares-mod.ts"
35+
);
3336

3437
let jsonResult: Record<string, unknown> | undefined;
3538
// skipcq: JS-0323 - Test mock requires minimal Context interface
@@ -49,7 +52,7 @@ Deno.test({
4952
};
5053

5154
try {
52-
await campaignBillMiddleware(context, async () => {});
55+
await createFinalResponseMiddleware(context, async () => {});
5356
} finally {
5457
Deno.env.get = originalEnv;
5558
}
@@ -59,20 +62,24 @@ Deno.test({
5962
}
6063
if (
6164
(jsonResult.data as Record<string, unknown>)?.error !==
62-
"Campaign creation data missing"
65+
"Campaign creation data missing"
6366
) {
6467
throw new Error(
65-
`Expected 'Campaign creation data missing', got: ${(jsonResult.data as Record<string, unknown>)?.error}`,
68+
`Expected 'Campaign creation data missing', got: ${
69+
(jsonResult.data as Record<string, unknown>)?.error
70+
}`,
6671
);
6772
}
6873
},
6974
});
7075

7176
Deno.test({
72-
name: "campaign-bill-middleware: should return success when billing disabled",
77+
name:
78+
"createFinalResponseMiddleware: should return success when billing disabled",
7379
async fn() {
74-
const { campaignBillMiddleware } =
75-
await import("../campaign-bill-middleware.ts");
80+
const { createFinalResponseMiddleware } = await import(
81+
"../middlewares-mod.ts"
82+
);
7683

7784
let jsonResult: Record<string, unknown> | undefined;
7885
// skipcq: JS-0323 - Test mock requires minimal Context interface
@@ -101,7 +108,7 @@ Deno.test({
101108
};
102109

103110
try {
104-
await campaignBillMiddleware(context, async () => {});
111+
await createFinalResponseMiddleware(context, async () => {});
105112
} finally {
106113
Deno.env.get = originalEnv;
107114
}
@@ -116,25 +123,31 @@ Deno.test({
116123
}
117124
if (
118125
(jsonResult.data as Record<string, unknown>)?.campaignId !==
119-
"campaign-123"
126+
"campaign-123"
120127
) {
121128
throw new Error(
122-
`Expected campaignId 'campaign-123', got: ${(jsonResult.data as Record<string, unknown>)?.campaignId}`,
129+
`Expected campaignId 'campaign-123', got: ${
130+
(jsonResult.data as Record<string, unknown>)?.campaignId
131+
}`,
123132
);
124133
}
125134
if ((jsonResult.data as Record<string, unknown>)?.queuedCount !== 10) {
126135
throw new Error(
127-
`Expected queuedCount 10, got: ${(jsonResult.data as Record<string, unknown>)?.queuedCount}`,
136+
`Expected queuedCount 10, got: ${
137+
(jsonResult.data as Record<string, unknown>)?.queuedCount
138+
}`,
128139
);
129140
}
130141
},
131142
});
132143

133144
Deno.test({
134-
name: "campaign-bill-middleware: should return success when charge fails (campaign exists)",
145+
name:
146+
"createFinalResponseMiddleware: should return success when charge fails (campaign exists)",
135147
async fn() {
136-
const { campaignBillMiddleware } =
137-
await import("../campaign-bill-middleware.ts");
148+
const { createFinalResponseMiddleware } = await import(
149+
"../middlewares-mod.ts"
150+
);
138151

139152
let jsonResult: Record<string, unknown> | undefined;
140153
// skipcq: JS-0323 - Test mock requires minimal Context interface
@@ -163,7 +176,7 @@ Deno.test({
163176
};
164177

165178
try {
166-
await campaignBillMiddleware(context, async () => {});
179+
await createFinalResponseMiddleware(context, async () => {});
167180
} finally {
168181
Deno.env.get = originalEnv;
169182
}
@@ -173,15 +186,19 @@ Deno.test({
173186
(jsonResult.data as Record<string, unknown>)?.msg !== "Campaign queued"
174187
) {
175188
throw new Error(
176-
`Expected success response even when billing fails, got: ${JSON.stringify(jsonResult)}`,
189+
`Expected success response even when billing fails, got: ${
190+
JSON.stringify(jsonResult)
191+
}`,
177192
);
178193
}
179194
if (
180195
(jsonResult.data as Record<string, unknown>)?.campaignId !==
181-
"campaign-456"
196+
"campaign-456"
182197
) {
183198
throw new Error(
184-
`Expected campaignId 'campaign-456', got: ${(jsonResult.data as Record<string, unknown>)?.campaignId}`,
199+
`Expected campaignId 'campaign-456', got: ${
200+
(jsonResult.data as Record<string, unknown>)?.campaignId
201+
}`,
185202
);
186203
}
187204
},

0 commit comments

Comments
 (0)