Skip to content

Commit 5bfa0e0

Browse files
author
Miriad
committed
Merge feat/wire-trend-research into dev — ingest cron uses 5-source trend discovery + optional NotebookLM research
2 parents 5a90d6f + 433ceff commit 5bfa0e0

File tree

1 file changed

+112
-97
lines changed

1 file changed

+112
-97
lines changed

‎app/api/cron/ingest/route.ts‎

Lines changed: 112 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,13 @@ import type { NextRequest } from "next/server";
44

55
import { generateWithGemini, stripCodeFences } from "@/lib/gemini";
66
import { writeClient } from "@/lib/sanity-write-client";
7+
import { discoverTrends, type TrendResult } from "@/lib/services/trend-discovery";
8+
import { conductResearch, type ResearchPayload } from "@/lib/services/research";
79

810
// ---------------------------------------------------------------------------
911
// Types
1012
// ---------------------------------------------------------------------------
1113

12-
interface RSSItem {
13-
title: string;
14-
url: string;
15-
}
16-
1714
interface ScriptScene {
1815
sceneNumber: number;
1916
sceneType: "narration" | "code" | "list" | "comparison" | "mockup";
@@ -62,122 +59,107 @@ interface CriticResult {
6259
}
6360

6461
// ---------------------------------------------------------------------------
65-
// RSS Feed Helpers
62+
// Fallback topics (used when discoverTrends returns empty)
6663
// ---------------------------------------------------------------------------
6764

68-
const RSS_FEEDS = [
69-
"https://hnrss.org/newest?q=javascript+OR+react+OR+nextjs+OR+typescript&points=50",
70-
"https://dev.to/feed/tag/webdev",
71-
];
72-
73-
const FALLBACK_TOPICS: RSSItem[] = [
65+
const FALLBACK_TRENDS: TrendResult[] = [
7466
{
75-
title: "React Server Components: The Future of Web Development",
76-
url: "https://react.dev/blog",
67+
topic: "React Server Components: The Future of Web Development",
68+
slug: "react-server-components",
69+
score: 80,
70+
signals: [{ source: "blog", title: "React Server Components", url: "https://react.dev/blog", score: 80 }],
71+
whyTrending: "Major shift in React architecture",
72+
suggestedAngle: "Explain what RSC changes for everyday React developers",
7773
},
7874
{
79-
title: "TypeScript 5.x: New Features Every Developer Should Know",
80-
url: "https://devblogs.microsoft.com/typescript/",
75+
topic: "TypeScript 5.x: New Features Every Developer Should Know",
76+
slug: "typescript-5x-features",
77+
score: 75,
78+
signals: [{ source: "blog", title: "TypeScript 5.x", url: "https://devblogs.microsoft.com/typescript/", score: 75 }],
79+
whyTrending: "New TypeScript release with major DX improvements",
80+
suggestedAngle: "Walk through the top 5 new features with code examples",
8181
},
8282
{
83-
title: "Next.js App Router Best Practices for 2025",
84-
url: "https://nextjs.org/blog",
83+
topic: "Next.js App Router Best Practices for 2025",
84+
slug: "nextjs-app-router-2025",
85+
score: 70,
86+
signals: [{ source: "blog", title: "Next.js App Router", url: "https://nextjs.org/blog", score: 70 }],
87+
whyTrending: "App Router adoption is accelerating",
88+
suggestedAngle: "Common pitfalls and how to avoid them",
8589
},
8690
{
87-
title: "The State of CSS in 2025: Container Queries, Layers, and More",
88-
url: "https://web.dev/blog",
91+
topic: "The State of CSS in 2025: Container Queries, Layers, and More",
92+
slug: "css-2025-state",
93+
score: 65,
94+
signals: [{ source: "blog", title: "CSS 2025", url: "https://web.dev/blog", score: 65 }],
95+
whyTrending: "CSS has gained powerful new features",
96+
suggestedAngle: "Demo the top 3 CSS features you should be using today",
8997
},
9098
{
91-
title: "WebAssembly is Changing How We Build Web Apps",
92-
url: "https://webassembly.org/",
99+
topic: "WebAssembly is Changing How We Build Web Apps",
100+
slug: "webassembly-web-apps",
101+
score: 60,
102+
signals: [{ source: "blog", title: "WebAssembly", url: "https://webassembly.org/", score: 60 }],
103+
whyTrending: "WASM adoption growing in production apps",
104+
suggestedAngle: "Real-world use cases where WASM outperforms JS",
93105
},
94106
];
95107

96-
function extractRSSItems(xml: string): RSSItem[] {
97-
const items: RSSItem[] = [];
98-
const itemRegex = /<item>([\s\S]*?)<\/item>/gi;
99-
let itemMatch: RegExpExecArray | null;
108+
// ---------------------------------------------------------------------------
109+
// Gemini Script Generation
110+
// ---------------------------------------------------------------------------
100111

101-
while ((itemMatch = itemRegex.exec(xml)) !== null) {
102-
const block = itemMatch[1];
112+
const SYSTEM_INSTRUCTION =
113+
"You are a content strategist for CodingCat.dev, a web development education channel. You create engaging, Cleo Abram-style explainer video scripts that are educational, energetic, and concise (60-90 seconds).";
103114

104-
const titleMatch = block.match(/<title><!\[CDATA\[(.*?)\]\]><\/title>/);
105-
const titleAlt = block.match(/<title>(.*?)<\/title>/);
106-
const title = titleMatch?.[1] ?? titleAlt?.[1] ?? "";
115+
function buildPrompt(trends: TrendResult[], research?: ResearchPayload): string {
116+
const topicList = trends
117+
.map((t, i) => `${i + 1}. "${t.topic}" (score: ${t.score}) — ${t.whyTrending}\n Sources: ${t.signals.map(s => s.url).join(", ")}`)
118+
.join("\n");
107119

108-
const linkMatch = block.match(/<link>(.*?)<\/link>/);
109-
const url = linkMatch?.[1] ?? "";
120+
// If we have research data, include it as enrichment
121+
let researchContext = "";
122+
if (research) {
123+
researchContext = `\n\n## Research Data (use this to create an informed, accurate script)\n\n`;
124+
researchContext += `### Briefing\n${research.briefing}\n\n`;
110125

111-
if (title && url) {
112-
items.push({ title: title.trim(), url: url.trim() });
126+
if (research.talkingPoints.length > 0) {
127+
researchContext += `### Key Talking Points\n${research.talkingPoints.map((tp, i) => `${i + 1}. ${tp}`).join("\n")}\n\n`;
113128
}
114-
}
115-
116-
return items;
117-
}
118129

119-
async function fetchTrendingTopics(): Promise<RSSItem[]> {
120-
const allItems: RSSItem[] = [];
130+
if (research.codeExamples.length > 0) {
131+
researchContext += `### Code Examples (use these in "code" scenes)\n`;
132+
for (const ex of research.codeExamples.slice(0, 5)) {
133+
researchContext += `\`\`\`${ex.language}\n${ex.snippet}\n\`\`\`\nContext: ${ex.context}\n\n`;
134+
}
135+
}
121136

122-
const results = await Promise.allSettled(
123-
RSS_FEEDS.map(async (feedUrl) => {
124-
const controller = new AbortController();
125-
const timeout = setTimeout(() => controller.abort(), 10_000);
126-
try {
127-
const res = await fetch(feedUrl, { signal: controller.signal });
128-
if (!res.ok) {
129-
console.warn(
130-
`[CRON/ingest] RSS fetch failed for ${feedUrl}: ${res.status}`,
131-
);
132-
return [];
137+
if (research.comparisonData && research.comparisonData.length > 0) {
138+
researchContext += `### Comparison Data (use in "comparison" scenes)\n`;
139+
for (const comp of research.comparisonData) {
140+
researchContext += `${comp.leftLabel} vs ${comp.rightLabel}:\n`;
141+
for (const row of comp.rows) {
142+
researchContext += ` - ${row.left} | ${row.right}\n`;
133143
}
134-
const xml = await res.text();
135-
return extractRSSItems(xml);
136-
} finally {
137-
clearTimeout(timeout);
144+
researchContext += "\n";
138145
}
139-
}),
140-
);
141-
142-
for (const result of results) {
143-
if (result.status === "fulfilled") {
144-
allItems.push(...result.value);
145-
} else {
146-
console.warn("[CRON/ingest] RSS feed error:", result.reason);
147146
}
148-
}
149147

150-
const seen = new Set<string>();
151-
const unique = allItems.filter((item) => {
152-
const key = item.title.toLowerCase();
153-
if (seen.has(key)) return false;
154-
seen.add(key);
155-
return true;
156-
});
148+
if (research.sceneHints.length > 0) {
149+
researchContext += `### Scene Type Suggestions\n`;
150+
for (const hint of research.sceneHints) {
151+
researchContext += `- ${hint.suggestedSceneType}: ${hint.reason}\n`;
152+
}
153+
}
157154

158-
if (unique.length === 0) {
159-
console.warn("[CRON/ingest] No RSS items fetched, using fallback topics");
160-
return FALLBACK_TOPICS;
155+
if (research.infographicPath) {
156+
researchContext += `\n### Infographic Available\nAn infographic has been generated for this topic. Use sceneType "narration" with bRollUrl pointing to the infographic for at least one scene.\n`;
157+
}
161158
}
162159

163-
return unique.slice(0, 10);
164-
}
165-
166-
// ---------------------------------------------------------------------------
167-
// Gemini Script Generation
168-
// ---------------------------------------------------------------------------
169-
170-
const SYSTEM_INSTRUCTION =
171-
"You are a content strategist for CodingCat.dev, a web development education channel. You create engaging, Cleo Abram-style explainer video scripts that are educational, energetic, and concise (60-90 seconds).";
172-
173-
function buildPrompt(topics: RSSItem[]): string {
174-
const topicList = topics
175-
.map((t, i) => `${i + 1}. "${t.title}" — ${t.url}`)
176-
.join("\n");
177-
178160
return `Here are today's trending web development topics:
179161
180-
${topicList}
162+
${topicList}${researchContext}
181163
182164
Pick the MOST interesting and timely topic for a short explainer video (60-90 seconds). Then generate a complete video script as JSON.
183165
@@ -334,6 +316,8 @@ Respond with ONLY the JSON object.`,
334316
async function createSanityDocuments(
335317
script: GeneratedScript,
336318
criticResult: CriticResult,
319+
trends: TrendResult[],
320+
research?: ResearchPayload,
337321
) {
338322
const isFlagged = criticResult.score < 50;
339323

@@ -369,6 +353,9 @@ async function createSanityDocuments(
369353
...(isFlagged && {
370354
flaggedReason: `Quality score ${criticResult.score}/100. Issues: ${criticResult.issues.join("; ") || "Low quality score"}`,
371355
}),
356+
trendScore: trends[0]?.score,
357+
trendSources: trends[0]?.signals.map(s => s.source).join(", "),
358+
researchNotebookId: research?.notebookId,
372359
});
373360

374361
console.log(`[CRON/ingest] Created automatedVideo: ${automatedVideo._id}`);
@@ -394,12 +381,38 @@ export async function GET(request: NextRequest) {
394381
}
395382

396383
try {
397-
console.log("[CRON/ingest] Fetching trending topics...");
398-
const topics = await fetchTrendingTopics();
399-
console.log(`[CRON/ingest] Found ${topics.length} topics`);
384+
// Step 1: Discover trending topics (replaces fetchTrendingTopics)
385+
console.log("[CRON/ingest] Discovering trending topics...");
386+
let trends: TrendResult[];
387+
try {
388+
trends = await discoverTrends({ lookbackDays: 7, maxTopics: 10 });
389+
console.log(`[CRON/ingest] Found ${trends.length} trending topics`);
390+
} catch (err) {
391+
console.warn("[CRON/ingest] Trend discovery failed, using fallback topics:", err);
392+
trends = [];
393+
}
394+
395+
// Fall back to hardcoded topics if discovery returns empty or failed
396+
if (trends.length === 0) {
397+
console.warn("[CRON/ingest] No trends discovered, using fallback topics");
398+
trends = FALLBACK_TRENDS;
399+
}
400+
401+
// Step 2: Optional deep research on top topic
402+
let research: ResearchPayload | undefined;
403+
if (process.env.ENABLE_NOTEBOOKLM_RESEARCH === "true") {
404+
console.log(`[CRON/ingest] Conducting research on: "${trends[0].topic}"...`);
405+
try {
406+
research = await conductResearch(trends[0].topic);
407+
console.log(`[CRON/ingest] Research complete: ${research.sources.length} sources, ${research.sceneHints.length} scene hints`);
408+
} catch (err) {
409+
console.warn("[CRON/ingest] Research failed, continuing without:", err);
410+
}
411+
}
400412

413+
// Step 3: Generate script with Gemini (enriched with research)
401414
console.log("[CRON/ingest] Generating script with Gemini...");
402-
const prompt = buildPrompt(topics);
415+
const prompt = buildPrompt(trends, research);
403416
const rawResponse = await generateWithGemini(prompt, SYSTEM_INSTRUCTION);
404417

405418
let script: GeneratedScript;
@@ -430,7 +443,7 @@ export async function GET(request: NextRequest) {
430443
);
431444

432445
console.log("[CRON/ingest] Creating Sanity documents...");
433-
const result = await createSanityDocuments(script, criticResult);
446+
const result = await createSanityDocuments(script, criticResult, trends, research);
434447

435448
console.log("[CRON/ingest] Done!", result);
436449

@@ -439,7 +452,9 @@ export async function GET(request: NextRequest) {
439452
...result,
440453
title: script.title,
441454
criticScore: criticResult.score,
442-
topicCount: topics.length,
455+
trendCount: trends.length,
456+
trendScore: trends[0]?.score,
457+
researchEnabled: !!research,
443458
});
444459
} catch (err) {
445460
console.error("[CRON/ingest] Unexpected error:", err);

0 commit comments

Comments
 (0)