Skip to content

Commit 5934068

Browse files
Miriadresearch
andcommitted
fix: trend discovery cache + auth JSON parsing + GitHub stars regex
1. Add cache: 'no-store' to fetchWithTimeout — Next.js extends fetch() with aggressive caching that was causing all sources to return 0 results 2. Handle double-encoded JSON in NOTEBOOKLM_AUTH_JSON — .env.local wraps values in quotes, producing a JSON string literal that needs double-parse 3. Fix GitHub stars regex to match "this week" and "this month" (not just "today") 4. Add verbose logging for Dev.to tag failures and empty blog feed parsing 5. Increase fetch timeout from 10s to 15s Co-authored-by: research <research@miriad.systems>
1 parent 6ec3fea commit 5934068

File tree

2 files changed

+25
-7
lines changed

2 files changed

+25
-7
lines changed

lib/services/notebooklm/auth.ts

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,11 +60,18 @@ function parseCookiesFromEnv(): Record<string, string> {
6060

6161
let storageState: { cookies?: NotebookLMCookie[] };
6262
try {
63-
storageState = JSON.parse(authJson) as { cookies?: NotebookLMCookie[] };
63+
// .env.local may double-quote the value, producing a JSON string literal.
64+
// Try parsing once; if the result is a string, parse again.
65+
let parsed: unknown = JSON.parse(authJson);
66+
if (typeof parsed === 'string') {
67+
parsed = JSON.parse(parsed);
68+
}
69+
storageState = parsed as { cookies?: NotebookLMCookie[] };
6470
} catch {
6571
throw new Error(
6672
'[NotebookLM] NOTEBOOKLM_AUTH_JSON is not valid JSON. ' +
67-
'Expected Playwright storage state format: {"cookies": [...]}'
73+
'Expected Playwright storage state format: {"cookies": [...]}. ' +
74+
'Tip: In .env.local, set it without wrapping quotes: NOTEBOOKLM_AUTH_JSON={"cookies":[...]}'
6875
);
6976
}
7077

lib/services/trend-discovery.ts

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -287,9 +287,14 @@ async function fetchWithTimeout(
287287
init?: RequestInit,
288288
): Promise<Response> {
289289
const controller = new AbortController();
290-
const timeout = setTimeout(() => controller.abort(), 10_000);
290+
const timeout = setTimeout(() => controller.abort(), 15_000);
291291
try {
292-
return await fetch(url, { ...init, signal: controller.signal });
292+
return await fetch(url, {
293+
...init,
294+
signal: controller.signal,
295+
// Next.js extends fetch() with aggressive caching — disable it
296+
cache: 'no-store' as RequestCache,
297+
});
293298
} finally {
294299
clearTimeout(timeout);
295300
}
@@ -550,7 +555,10 @@ async function fetchDevto(lookbackDays: number): Promise<TrendSignal[]> {
550555
const res = await fetchWithTimeout(
551556
`https://dev.to/api/articles?tag=${tag}&top=${lookbackDays}`,
552557
);
553-
if (!res.ok) return [];
558+
if (!res.ok) {
559+
console.warn(`${LOG_PREFIX} Dev.to tag ${tag} failed: ${res.status}`);
560+
return [];
561+
}
554562
return (await res.json()) as DevtoArticle[];
555563
}),
556564
);
@@ -613,6 +621,9 @@ async function fetchBlogFeeds(lookbackDays: number): Promise<TrendSignal[]> {
613621
}
614622
const xml = await res.text();
615623
const items = parseFeedItems(xml);
624+
if (items.length === 0) {
625+
console.warn(`${LOG_PREFIX} Blog feed ${feed.name}: parsed 0 items from ${xml.length} bytes`);
626+
}
616627
return items.map((item) => ({ ...item, feedName: feed.name }));
617628
}),
618629
);
@@ -776,9 +787,9 @@ function parseGitHubTrendingHTML(html: string, language: string): GitHubTrending
776787
.replace(/<[^>]+>/g, "")
777788
.trim();
778789

779-
// Stars today from "N stars today" text
790+
// Stars from "N stars today" or "N stars this week" text
780791
const starsMatch = block.match(
781-
/(\d[\d,]*)\s+stars?\s+today/i,
792+
/(\d[\d,]*)\s+stars?\s+(?:today|this\s+week|this\s+month)/i,
782793
);
783794
const starsToday = starsMatch
784795
? parseInt(starsMatch[1].replace(/,/g, ""), 10)

0 commit comments

Comments
 (0)