Skip to content

Commit 35b14ef

Browse files
steven-teysteven-tey
authored
Merge pull request dubinc#2373 from dubinc/framer-backfill
Add Framer backfill remainder scripts
2 parents c8fa6e6 + c8568dd commit 35b14ef

2 files changed

Lines changed: 299 additions & 0 deletions

File tree

‎apps/web/scripts/framer/backfill-remainder.ts‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
import { prisma } from "@dub/prisma";
2+
import "dotenv-flow/config";
3+
import * as fs from "fs";
4+
import * as Papa from "papaparse";
5+
import { tb } from "../../lib/tinybird/client";
6+
import z from "../../lib/zod";
7+
8+
const getFramerLeadEvents = tb.buildPipe({
9+
pipe: "get_framer_lead_events",
10+
parameters: z.object({
11+
linkId: z.string(),
12+
}),
13+
data: z.any(),
14+
});
15+
16+
const framerCombinedData: {
17+
via: string;
18+
externalId: string;
19+
eventName: string;
20+
creationDate: string;
21+
}[] = [];
22+
const externalIdEventNameSet = new Set<string>();
23+
24+
async function processFramerData(linkToBackfill: {
25+
via: string;
26+
linkId: string;
27+
}) {
28+
return new Promise<void>((resolve) => {
29+
Papa.parse(fs.createReadStream("framer_combined.csv", "utf-8"), {
30+
header: true,
31+
skipEmptyLines: true,
32+
step: (result: {
33+
data: {
34+
via: string;
35+
externalId: string;
36+
eventName: string;
37+
creationDate: string;
38+
};
39+
}) => {
40+
if (linkToBackfill.via === result.data.via) {
41+
// check if externalId:eventName pair already exists in framerCombinedData
42+
if (
43+
externalIdEventNameSet.has(
44+
`${result.data.externalId}:${result.data.eventName}`,
45+
)
46+
) {
47+
return;
48+
}
49+
externalIdEventNameSet.add(
50+
`${result.data.externalId}:${result.data.eventName}`,
51+
);
52+
framerCombinedData.push({
53+
via: result.data.via,
54+
externalId: result.data.externalId,
55+
eventName: result.data.eventName,
56+
creationDate: result.data.creationDate,
57+
});
58+
}
59+
},
60+
complete: async () => {
61+
console.log(
62+
`Found ${framerCombinedData.length} framerCombinedData for ${linkToBackfill.via} (${linkToBackfill.linkId})`,
63+
);
64+
65+
const { data: leadEvents } = await getFramerLeadEvents({
66+
linkId: linkToBackfill.linkId,
67+
});
68+
69+
const customerIdsSet = new Set(
70+
leadEvents.map((event) => event.customer_id),
71+
);
72+
73+
const customers = await prisma.customer.findMany({
74+
where: {
75+
id: {
76+
in: Array.from(customerIdsSet),
77+
},
78+
},
79+
select: {
80+
id: true,
81+
externalId: true,
82+
},
83+
});
84+
85+
const processedLeadEvents = leadEvents.map((event) => {
86+
const customer = customers.find(
87+
(customer) => customer.id === event.customer_id,
88+
)!;
89+
return {
90+
linkId: event.link_id,
91+
externalId: customer.externalId!,
92+
eventName: event.event_name,
93+
creationDate: event.timestamp,
94+
};
95+
});
96+
97+
console.log(
98+
`Found ${processedLeadEvents.length} processedLeadEvents for ${linkToBackfill.via} (${linkToBackfill.linkId})`,
99+
);
100+
101+
// find events in framerCombinedData that don't exist in processedLeadEvents
102+
const eventsToBackfill = framerCombinedData.filter(
103+
(event) =>
104+
!processedLeadEvents.find(
105+
(ple) =>
106+
ple.externalId.toLowerCase() ===
107+
event.externalId.toLowerCase() &&
108+
ple.eventName.toLowerCase() === event.eventName.toLowerCase(),
109+
),
110+
);
111+
112+
console.log(
113+
`Found ${eventsToBackfill.length} events to backfill for ${linkToBackfill.via} (${linkToBackfill.linkId})`,
114+
);
115+
// append to framer_remaining_events_final.csv
116+
fs.appendFileSync(
117+
"framer_remaining_events_final.csv",
118+
Papa.unparse(eventsToBackfill),
119+
);
120+
resolve();
121+
},
122+
});
123+
});
124+
}
125+
126+
async function main() {
127+
const linksWithSales = JSON.parse(
128+
fs.readFileSync("framer_links_with_sales_to_backfill.json", "utf-8"),
129+
);
130+
131+
if (linksWithSales.length === 0) {
132+
console.log("No more links to process!");
133+
return;
134+
}
135+
136+
// Get the first link to process
137+
const linkToProcess = linksWithSales[0];
138+
139+
if (linkToProcess.linkId) {
140+
console.log(
141+
`Processing link: ${linkToProcess.via} (${linkToProcess.linkId})`,
142+
);
143+
144+
await processFramerData(linkToProcess);
145+
} else {
146+
console.log("No linkId found for linkToProcess");
147+
}
148+
149+
// Remove the processed link from the array
150+
linksWithSales.shift();
151+
152+
// Write the updated array back to the file
153+
fs.writeFileSync(
154+
"framer_links_with_sales_to_backfill.json",
155+
JSON.stringify(linksWithSales, null, 2),
156+
);
157+
158+
console.log(`Remaining links to process: ${linksWithSales.length}`);
159+
}
160+
161+
main();

‎apps/web/scripts/framer/get-remaining-links-to-backfill.ts‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
import { prisma } from "@dub/prisma";
2+
import "dotenv-flow/config";
3+
import * as fs from "fs";
4+
import * as Papa from "papaparse";
5+
6+
const framerCombinedData: {
7+
via: string;
8+
externalId: string;
9+
eventName: string;
10+
creationDate: string;
11+
}[] = [];
12+
const externalIdEventNameSet = new Set<string>();
13+
14+
const salesData: Record<string, number> = {};
15+
16+
async function main() {
17+
// First read the sales data
18+
Papa.parse(fs.createReadStream("framer_links_with_sales.csv", "utf-8"), {
19+
header: true,
20+
skipEmptyLines: true,
21+
step: (result: {
22+
data: {
23+
link_id: string;
24+
sales: string;
25+
};
26+
}) => {
27+
salesData[result.data.link_id] = parseInt(result.data.sales);
28+
},
29+
complete: () => {
30+
console.log(
31+
`Found ${Object.keys(salesData).length} links with sales data`,
32+
);
33+
// After sales data is loaded, process the combined data
34+
processCombinedData();
35+
},
36+
});
37+
}
38+
39+
function processCombinedData() {
40+
Papa.parse(fs.createReadStream("framer_combined.csv", "utf-8"), {
41+
header: true,
42+
skipEmptyLines: true,
43+
step: (result: {
44+
data: {
45+
via: string;
46+
externalId: string;
47+
eventName: string;
48+
creationDate: string;
49+
};
50+
}) => {
51+
// check if externalId:eventName pair already exists in framerCombinedData
52+
if (
53+
externalIdEventNameSet.has(
54+
`${result.data.externalId}:${result.data.eventName}`,
55+
)
56+
) {
57+
return;
58+
}
59+
externalIdEventNameSet.add(
60+
`${result.data.externalId}:${result.data.eventName}`,
61+
);
62+
framerCombinedData.push({
63+
via: result.data.via,
64+
externalId: result.data.externalId,
65+
eventName: result.data.eventName,
66+
creationDate: result.data.creationDate,
67+
});
68+
},
69+
complete: async () => {
70+
console.log(`Found ${framerCombinedData.length} links in Framer`);
71+
72+
// convert to via + count
73+
const viaCount = framerCombinedData.reduce(
74+
(acc, curr) => {
75+
acc[curr.via] = (acc[curr.via] || 0) + 1;
76+
return acc;
77+
},
78+
{} as Record<string, number>,
79+
);
80+
81+
// order by count
82+
const viaCountArray = Object.entries(viaCount)
83+
.map(([via, count]) => ({
84+
via,
85+
count,
86+
}))
87+
.sort((a, b) => b.count - a.count);
88+
89+
const prismaLinks = await prisma.link.findMany({
90+
where: {
91+
domain: "framer.link",
92+
key: {
93+
in: viaCountArray.map(({ via }) => via),
94+
},
95+
},
96+
select: {
97+
id: true,
98+
key: true,
99+
},
100+
});
101+
102+
const linkIdsWithCount = viaCountArray
103+
.map(({ via }) => {
104+
const link = prismaLinks.find((link) => link.key === via);
105+
const count = viaCount[via];
106+
const sales = link ? salesData[link.id] || 0 : 0;
107+
if (count === sales) {
108+
return null;
109+
}
110+
return {
111+
linkId: link?.id,
112+
via,
113+
count,
114+
recordedSales: sales,
115+
};
116+
})
117+
.filter(Boolean);
118+
119+
// Sort by sales instead of count
120+
linkIdsWithCount.sort(
121+
(a, b) => (b?.recordedSales || 0) - (a?.recordedSales || 0),
122+
);
123+
124+
console.log(
125+
`Found ${linkIdsWithCount.length} links with sales data that need to be backfilled`,
126+
);
127+
128+
console.table(linkIdsWithCount.slice(0, 100));
129+
130+
fs.writeFileSync(
131+
"framer_links_with_sales_to_backfill.json",
132+
JSON.stringify(linkIdsWithCount, null, 2),
133+
);
134+
},
135+
});
136+
}
137+
138+
main();

0 commit comments

Comments
 (0)