Skip to content

Commit 67be9b7

Browse files
claudeclaude
committed
feat: add incremental Discord channel scanning to research pipeline
Extends the daily research workflow to also scan the Cloudflare Developers Discord alongside GitHub, tracking new messages per channel in D1 and surfacing highlights in the daily newsletter email. Key changes ─────────── • backend/src/db/schemas/discord/index.ts — two new tables: - discord_scan_log tracks the last scanned message ID per channel - discord_messages stores every ingested message for deduplication and AI analysis (score 0-100, one-sentence summary, category) • migrations/core/0059_discord_scan.sql — SQL DDL for both tables • backend/src/workflows/discord.ts — DiscordResearchWorkflow class + runDiscordResearch() helper: - discovers all text/forum channels across every guild the bot is in - compares channel latest-message date against discord_scan_log to find only new content (incremental, never re-analyses old messages) - classifies each channel into one of four categories: what-i-built, announcement, binding, general - batches new messages through Worker AI for scoring + summarisation - persists high-scoring items (≥60/100) as research_candidates so they flow through the existing HITL review loop - updates discord_scan_log with the newest message ID after each scan • backend/src/workflows/index.ts — exports DiscordResearchWorkflow • backend/src/index.ts - exports DiscordResearchWorkflow alongside other workflow classes - runs runDiscordResearch() as a ctx.waitUntil task on the 0 9 * * * cron trigger (same daily window as GitHub trending research) - sends a structured Discord digest email when highlights are found • wrangler.jsonc — registers the discord-research-workflow binding (DISCORD_RESEARCH_WORKFLOW / DiscordResearchWorkflow) • backend/src/utils/email/send/repo-discovery.ts — fixed double-compile bug: now compiles the Handlebars template in a single pass so both contentHtml and dailyTrendsData fields render correctly together • backend/src/utils/email/templates/base/email-fallback.hbs — added {{#if summary}} banner and {{#each repos}} card loop so structured digest data renders properly; also added Discord-friendly styles https://claude.ai/code/session_01FmX8SgoUgetVsTNEM6b6Wh
1 parent 61a6d23 commit 67be9b7

9 files changed

Lines changed: 806 additions & 18 deletions

File tree

‎backend/src/db/schemas/discord/index.ts‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
import { sqliteTable, text, integer, index } from "drizzle-orm/sqlite-core";
2+
import { sql } from "drizzle-orm";
3+
import { createId } from "@paralleldrive/cuid2";
4+
5+
/**
6+
* Tracks the last-scanned state for each Discord channel so the scanner
7+
* knows exactly where to resume from on the next run.
8+
*/
9+
export const discordScanLog = sqliteTable(
10+
"discord_scan_log",
11+
{
12+
id: text("id").primaryKey().$defaultFn(() => createId()),
13+
guildId: text("guild_id").notNull(),
14+
channelId: text("channel_id").notNull().unique(),
15+
channelName: text("channel_name"),
16+
/** The Discord message Snowflake ID of the last message we ingested. */
17+
lastMessageId: text("last_message_id"),
18+
/** Unix epoch (seconds) of the last successful scan for this channel. */
19+
lastScannedAt: integer("last_scanned_at", { mode: "timestamp" }).notNull(),
20+
createdAt: integer("created_at", { mode: "timestamp" })
21+
.default(sql`(strftime('%s', 'now'))`)
22+
.notNull(),
23+
updatedAt: integer("updated_at", { mode: "timestamp" })
24+
.default(sql`(strftime('%s', 'now'))`)
25+
.notNull(),
26+
},
27+
(t) => ({
28+
guildIdx: index("discord_scan_log_guild_id_idx").on(t.guildId),
29+
})
30+
);
31+
32+
/**
33+
* Stores every Discord message we have ingested so we can:
34+
* - avoid re-analysing content we have already seen
35+
* - look up a message by its Discord ID for deduplication
36+
* - surface insights through the research pipeline
37+
*/
38+
export const discordMessages = sqliteTable(
39+
"discord_messages",
40+
{
41+
/** Discord Snowflake ID — used as the primary key for deduplication. */
42+
id: text("id").primaryKey(),
43+
guildId: text("guild_id").notNull(),
44+
channelId: text("channel_id").notNull(),
45+
channelName: text("channel_name"),
46+
authorId: text("author_id"),
47+
authorUsername: text("author_username"),
48+
content: text("content").notNull(),
49+
/** ISO-8601 timestamp returned by Discord. */
50+
discordTimestamp: text("discord_timestamp").notNull(),
51+
/**
52+
* Coarse category derived from the channel name:
53+
* 'what-i-built' | 'announcement' | 'binding' | 'general'
54+
*/
55+
category: text("category", {
56+
enum: ["what-i-built", "announcement", "binding", "general"],
57+
})
58+
.default("general")
59+
.notNull(),
60+
/** AI-assigned interest score 0–100. NULL until analysed. */
61+
aiScore: integer("ai_score"),
62+
/** One-sentence AI summary / reasoning. NULL until analysed. */
63+
aiSummary: text("ai_summary"),
64+
/** Whether this message has been through the AI analysis pass. */
65+
analysed: integer("analysed", { mode: "boolean" }).default(false).notNull(),
66+
ingestedAt: integer("ingested_at", { mode: "timestamp" })
67+
.default(sql`(strftime('%s', 'now'))`)
68+
.notNull(),
69+
},
70+
(t) => ({
71+
channelIdx: index("discord_messages_channel_id_idx").on(t.channelId),
72+
guildIdx: index("discord_messages_guild_id_idx").on(t.guildId),
73+
categoryIdx: index("discord_messages_category_idx").on(t.category),
74+
analysedIdx: index("discord_messages_analysed_idx").on(t.analysed),
75+
})
76+
);

‎backend/src/db/schemas/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
*/
88
export * from './agents';
99
export * from './app';
10+
export * from './discord';
1011
export * from './containers';
1112
export * from './github';
1213
export * from './jules';

‎backend/src/index.ts‎

Lines changed: 78 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -698,7 +698,7 @@ export { Supervisor } from "@/ai/agents/Supervisor";
698698
export { DeepReasoningAgent } from "@/ai/agents/DeepReasoning";
699699
// export { DataProcessor } from "@/do/DataProcessor";
700700
export { GeminiAgent } from "@/ai/agents/Gemini";
701-
export { GithubSearchWorkflow, DeepResearchWorkflow, TopicResearchWorkflow } from "@/workflows";
701+
export { GithubSearchWorkflow, DeepResearchWorkflow, TopicResearchWorkflow, DiscordResearchWorkflow } from "@/workflows";
702702

703703
// Research Agents (Topic Research)
704704
export { TopicOrchestratorAgent } from "./ai/agents/TopicOrchestrator";
@@ -724,6 +724,83 @@ export { Sandbox } from '@cloudflare/sandbox'
724724
async function handleScheduled(event: ScheduledController, env: Env, ctx: ExecutionContext) {
725725
console.log('[Scheduled] Cron trigger fired:', event.cron);
726726

727+
// Daily Discord scan at 9 AM UTC (runs alongside GitHub research)
728+
if (event.cron === '0 9 * * *') {
729+
ctx.waitUntil(
730+
(async () => {
731+
try {
732+
console.log('[Scheduled] Starting daily Discord scan...');
733+
const { runDiscordResearch } = await import('@/workflows/discord');
734+
const discordResult = await runDiscordResearch(env, {});
735+
736+
if (discordResult.digest.length > 0 && env.SEND_EMAIL_NEWSLETTER) {
737+
// Group digest items by category for a structured email section
738+
const byCategory: Record<string, typeof discordResult.digest> = {};
739+
for (const item of discordResult.digest) {
740+
if (!byCategory[item.category]) byCategory[item.category] = [];
741+
byCategory[item.category].push(item);
742+
}
743+
744+
const categoryLabels: Record<string, string> = {
745+
'what-i-built': '🏗️ What People Are Building',
746+
'announcement': '📢 Cloudflare Announcements',
747+
'binding': '🔧 Tips & Tricks from Product Channels',
748+
'general': '💬 Notable Discussions',
749+
};
750+
751+
const sections = Object.entries(byCategory)
752+
.map(([cat, items]) => {
753+
const label = categoryLabels[cat] ?? cat;
754+
const bullets = items
755+
.sort((a, b) => b.aiScore - a.aiScore)
756+
.slice(0, 5)
757+
.map(
758+
(i) =>
759+
`<li><strong>@${i.author}</strong> in #${i.channelName}: ${i.aiSummary} <em>(score: ${i.aiScore}/100)</em></li>`
760+
)
761+
.join('');
762+
return `<h3>${label}</h3><ul>${bullets}</ul>`;
763+
})
764+
.join('');
765+
766+
const contentHtml = `
767+
<h2>Discord Highlights — ${new Date().toLocaleDateString()}</h2>
768+
<p>Scanned ${discordResult.ingested} new message(s) across Cloudflare Developers Discord channels. Found <strong>${discordResult.highlighted}</strong> noteworthy item(s).</p>
769+
${sections}
770+
`;
771+
772+
await sendRepoDiscoveryEmail(env, {
773+
subject: `Discord Digest — ${discordResult.highlighted} highlights — ${new Date().toLocaleDateString()}`,
774+
title: 'Cloudflare Discord Daily Digest',
775+
contentHtml,
776+
dailyTrendsData: {
777+
date: new Date().toLocaleDateString(),
778+
trend_summary: `${discordResult.highlighted} Discord highlights from ${discordResult.ingested} new messages scanned across the Cloudflare Developers server.`,
779+
top_picks: discordResult.digest
780+
.sort((a, b) => b.aiScore - a.aiScore)
781+
.slice(0, 10)
782+
.map((item) => ({
783+
name: `#${item.channelName} — @${item.author}`,
784+
url: `https://discord.com/channels/@me/${item.messageId}`,
785+
category: item.category,
786+
why_its_interesting: item.aiSummary,
787+
innovation_score: Math.round(item.aiScore / 10),
788+
})),
789+
},
790+
plainTextFallback: `Discord Digest: ${discordResult.highlighted} highlights found from ${discordResult.ingested} new messages.`,
791+
});
792+
793+
console.log('[Scheduled] Discord digest email sent.');
794+
} else {
795+
console.log(`[Scheduled] Discord scan complete — no highlights to email (ingested: ${discordResult.ingested}).`);
796+
}
797+
} catch (err) {
798+
console.error('[Scheduled] Discord scan failed:', err);
799+
}
800+
})()
801+
);
802+
}
803+
727804
// Daily research scan at 9 AM UTC
728805
if (event.cron === '0 9 * * *') {
729806
console.log('[Scheduled] Starting daily research scan...');

‎backend/src/utils/email/send/repo-discovery.ts‎

Lines changed: 12 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -63,25 +63,22 @@ export async function sendRepoDiscoveryEmail(env: Env, params: SendEmailParams):
6363
});
6464
}
6565

66-
// Import new fallback template if needed
67-
let finalContentHtml = params.contentHtml || "";
68-
66+
// Compile the fallback template once and pass all data in a single pass.
67+
// When dailyTrendsData is provided, the template renders the structured
68+
// {{#each repos}} digest. contentHtml (if set) is rendered via {{{contentHtml}}}.
69+
const fallbackTemplate = Handlebars.compile(fallbackEmailRaw);
70+
const templateContext: Record<string, unknown> = {
71+
subject: params.subject,
72+
title: params.title,
73+
contentHtml: params.contentHtml || "",
74+
};
6975
if (params.dailyTrendsData) {
70-
const digestTemplate = Handlebars.compile(fallbackEmailRaw);
71-
72-
// Map the external data property to the Handlebars variables
73-
finalContentHtml = digestTemplate({
74-
title: params.title,
75-
summary: params.dailyTrendsData.trend_summary,
76-
repos: params.dailyTrendsData.top_picks
77-
});
76+
templateContext.summary = params.dailyTrendsData.trend_summary;
77+
templateContext.repos = params.dailyTrendsData.top_picks;
7878
}
79-
80-
// Compile and append the standard HTML fallback
81-
const fallbackTemplate = Handlebars.compile(fallbackEmailRaw);
8279
msg.addMessage({
8380
contentType: "text/html",
84-
data: fallbackTemplate({ subject: params.subject, contentHtml: finalContentHtml }),
81+
data: fallbackTemplate(templateContext),
8582
});
8683

8784
// If AMP data is provided, compile and append the interactive layer

‎backend/src/utils/email/templates/base/email-fallback.hbs‎

Lines changed: 93 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
<head>
55
<meta charset="utf-8">
66
<meta name="viewport" content="width=device-width, initial-scale=1.0">
7-
<title>{{subject}}</title>
7+
<title>{{subject}}{{title}}</title>
88
<style>
99
/* Gmail-friendly resets */
1010
body,
@@ -37,6 +37,65 @@
3737
width: 100% !important;
3838
background-color: #f9fafb;
3939
}
40+
41+
.pick-card {
42+
border: 1px solid #e5e7eb;
43+
border-radius: 8px;
44+
padding: 16px;
45+
margin-bottom: 12px;
46+
background: #fafafa;
47+
}
48+
49+
.pick-name {
50+
font-weight: 600;
51+
font-size: 15px;
52+
color: #111827;
53+
margin: 0 0 4px 0;
54+
}
55+
56+
.pick-meta {
57+
font-size: 12px;
58+
color: #9ca3af;
59+
margin: 0 0 8px 0;
60+
}
61+
62+
.pick-reason {
63+
font-size: 14px;
64+
color: #374151;
65+
margin: 0;
66+
line-height: 1.5;
67+
}
68+
69+
.score-badge {
70+
display: inline-block;
71+
background: #f0fdf4;
72+
color: #16a34a;
73+
border-radius: 999px;
74+
padding: 2px 10px;
75+
font-size: 12px;
76+
font-weight: 600;
77+
margin-left: 8px;
78+
}
79+
80+
.section-header {
81+
font-size: 18px;
82+
font-weight: 700;
83+
color: #111827;
84+
margin: 28px 0 12px 0;
85+
padding-bottom: 8px;
86+
border-bottom: 2px solid #f59e0b;
87+
}
88+
89+
.summary-box {
90+
background: #fffbeb;
91+
border-left: 4px solid #f59e0b;
92+
padding: 12px 16px;
93+
border-radius: 4px;
94+
margin-bottom: 20px;
95+
font-size: 14px;
96+
color: #92400e;
97+
line-height: 1.6;
98+
}
4099
</style>
41100
</head>
42101

@@ -53,11 +112,43 @@
53112
<h1
54113
style="color: #ffffff; margin: 0; font-size: 24px; font-weight: 600; letter-spacing: -0.5px;">
55114
Agentic Research Team</h1>
115+
{{#if title}}
116+
<p style="color: #9ca3af; margin: 8px 0 0 0; font-size: 14px;">{{title}}</p>
117+
{{/if}}
56118
</td>
57119
</tr>
58120
<tr>
59121
<td style="padding: 40px 30px; color: #374151; font-size: 16px; line-height: 1.6;">
122+
123+
{{! Pre-built HTML block (highest priority) }}
124+
{{#if contentHtml}}
60125
{{{contentHtml}}}
126+
{{/if}}
127+
128+
{{! Structured digest — rendered when dailyTrendsData is passed }}
129+
{{#if summary}}
130+
<div class="summary-box">{{summary}}</div>
131+
{{/if}}
132+
133+
{{#if repos}}
134+
{{#each repos}}
135+
<div class="pick-card">
136+
<p class="pick-name">
137+
<a href="{{url}}" style="color: #1d4ed8; text-decoration: none;">{{name}}</a>
138+
{{#if innovation_score}}
139+
<span class="score-badge">{{innovation_score}}/10</span>
140+
{{/if}}
141+
</p>
142+
{{#if category}}
143+
<p class="pick-meta">{{category}}</p>
144+
{{/if}}
145+
{{#if why_its_interesting}}
146+
<p class="pick-reason">{{why_its_interesting}}</p>
147+
{{/if}}
148+
</div>
149+
{{/each}}
150+
{{/if}}
151+
61152
</td>
62153
</tr>
63154
<tr>
@@ -74,4 +165,4 @@
74165
</table>
75166
</body>
76167

77-
</html>
168+
</html>

0 commit comments

Comments
 (0)