Skip to content

Commit a36b873

Browse files
tannerlinsleytannerlinsley
committed
scheduled functions
1 parent 3dda470 commit a36b873

3 files changed

Lines changed: 104 additions & 205 deletions

File tree

‎netlify/functions/refresh-stats-cache.ts‎

Lines changed: 62 additions & 109 deletions
Original file line numberDiff line numberDiff line change
@@ -1,183 +1,136 @@
1-
import type { Handler, HandlerEvent, HandlerContext } from '@netlify/functions'
1+
import type { Config } from "@netlify/functions";
22
import {
33
refreshNpmOrgStats,
44
fetchGitHubOwnerStats,
55
fetchGitHubRepoStats,
6-
} from '~/utils/stats.functions'
7-
import { setCachedGitHubStats } from '~/utils/stats-db.server'
6+
} from "~/utils/stats.functions";
7+
import { setCachedGitHubStats } from "~/utils/stats-db.server";
88

99
/**
10-
* Netlify Background + Scheduled Function - Refresh org-level stats cache
11-
*
12-
* This function combines both background and scheduled execution:
13-
* - Can be invoked via HTTP POST (returns 202 immediately, runs in background)
14-
* - Can be triggered on a schedule (runs automatically via cron)
15-
*
16-
* Background functions have a 15-minute timeout (vs 30 seconds for regular functions)
17-
* and return a 202 response immediately while processing continues in the background.
10+
* Netlify Scheduled Function - Refresh org-level stats cache
1811
*
1912
* This function refreshes pre-aggregated stats for the TanStack organization:
2013
* - GitHub: stars, contributors, dependents
2114
* - NPM: total downloads across all packages (including legacy packages)
2215
*
23-
* Scheduled: Runs automatically every 6 hours (configured via export config)
24-
* Only called by Netlify's scheduler - not publicly accessible
25-
* Timeout: 15 minutes
16+
* Scheduled: Runs automatically every 6 hours
2617
* Concurrency: 8 packages at a time, 500ms delay between packages
2718
*
2819
* Performance Impact:
2920
* - Refresh time: ~10-20 minutes for 203 packages (199 scoped + 4 legacy)
3021
* - After refresh: ~100-200ms per request (served from cache)
3122
*/
32-
export const handler: Handler = async (
33-
event: HandlerEvent,
34-
context: HandlerContext,
35-
) => {
36-
console.log('[refresh-stats-cache-background] Starting stats refresh...')
23+
const handler = async (req: Request) => {
24+
const { next_run } = await req.json();
25+
26+
console.log("[refresh-stats-cache-background] Starting stats refresh...");
3727

38-
const startTime = Date.now()
28+
const startTime = Date.now();
3929

4030
try {
41-
const org = 'tanstack'
31+
const org = "tanstack";
4232

4333
// Refresh NPM org stats (fetches all packages, aggregates, and caches)
44-
console.log('[refresh-stats-cache-background] Refreshing NPM org stats...')
45-
const npmStats = await refreshNpmOrgStats(org)
34+
console.log("[refresh-stats-cache-background] Refreshing NPM org stats...");
35+
const npmStats = await refreshNpmOrgStats(org);
4636

4737
// Refresh GitHub org stats
4838
console.log(
49-
'[refresh-stats-cache-background] Refreshing GitHub org stats...',
50-
)
51-
const githubCacheKey = `org:${org}`
52-
const githubStats = await fetchGitHubOwnerStats(org)
53-
await setCachedGitHubStats(githubCacheKey, githubStats, 1)
39+
"[refresh-stats-cache-background] Refreshing GitHub org stats..."
40+
);
41+
const githubCacheKey = `org:${org}`;
42+
const githubStats = await fetchGitHubOwnerStats(org);
43+
await setCachedGitHubStats(githubCacheKey, githubStats, 1);
5444

5545
// Refresh GitHub stats for each library repo
5646
console.log(
57-
'[refresh-stats-cache-background] Refreshing GitHub stats for individual libraries...',
58-
)
59-
const { libraries } = await import('~/libraries')
47+
"[refresh-stats-cache-background] Refreshing GitHub stats for individual libraries..."
48+
);
49+
const { libraries } = await import("~/libraries");
6050
console.log(
6151
`[refresh-stats-cache-background] Found ${libraries.length} libraries to process:`,
62-
libraries.map((lib) => ({ id: lib.id, repo: lib.repo })),
63-
)
64-
const libraryResults = []
65-
const libraryErrors = []
52+
libraries.map((lib) => ({ id: lib.id, repo: lib.repo }))
53+
);
54+
const libraryResults = [];
55+
const libraryErrors = [];
6656

6757
for (let i = 0; i < libraries.length; i++) {
68-
const library = libraries[i]
58+
const library = libraries[i];
6959
if (!library.repo) {
7060
console.log(
71-
`[refresh-stats-cache-background] Skipping library ${library.id} - no repo`,
72-
)
73-
continue
61+
`[refresh-stats-cache-background] Skipping library ${library.id} - no repo`
62+
);
63+
continue;
7464
}
7565

7666
console.log(
77-
`[refresh-stats-cache-background] Processing library ${library.id} (${library.repo})...`,
78-
)
67+
`[refresh-stats-cache-background] Processing library ${library.id} (${library.repo})...`
68+
);
7969
try {
80-
const repoStats = await fetchGitHubRepoStats(library.repo)
70+
const repoStats = await fetchGitHubRepoStats(library.repo);
8171
console.log(
8272
`[refresh-stats-cache-background] Fetched stats for ${
8373
library.repo
8474
}: ${repoStats.starCount} stars, ${
8575
repoStats.contributorCount
86-
} contributors, ${repoStats.dependentCount ?? 'N/A'} dependents`,
87-
)
88-
await setCachedGitHubStats(library.repo, repoStats, 1)
76+
} contributors, ${repoStats.dependentCount ?? "N/A"} dependents`
77+
);
78+
await setCachedGitHubStats(library.repo, repoStats, 1);
8979
console.log(
90-
`[refresh-stats-cache-background] ✓ Successfully cached stats for ${library.repo}`,
91-
)
80+
`[refresh-stats-cache-background] ✓ Successfully cached stats for ${library.repo}`
81+
);
9282
libraryResults.push({
9383
repo: library.repo,
9484
stars: repoStats.starCount,
95-
})
85+
});
9686

9787
// Add delay between requests to avoid rate limiting (except for last item)
9888
if (i < libraries.length - 1) {
99-
await new Promise((resolve) => setTimeout(resolve, 500))
89+
await new Promise((resolve) => setTimeout(resolve, 500));
10090
}
10191
} catch (error) {
10292
const errorMessage =
103-
error instanceof Error ? error.message : String(error)
93+
error instanceof Error ? error.message : String(error);
10494
console.error(
10595
`[refresh-stats-cache-background] Failed to refresh ${library.repo}:`,
106-
errorMessage,
107-
)
96+
errorMessage
97+
);
10898
libraryErrors.push({
10999
repo: library.repo,
110100
error: errorMessage,
111-
})
101+
});
112102
}
113103
}
114104

115-
const duration = Date.now() - startTime
105+
const duration = Date.now() - startTime;
116106
console.log(
117107
`[refresh-stats-cache-background] ✓ Completed in ${duration}ms - NPM: ${npmStats.totalDownloads.toLocaleString()} downloads (${
118108
Object.keys(npmStats.packageStats || {}).length
119109
} packages), GitHub Org: ${githubStats.starCount.toLocaleString()} stars, Libraries: ${
120110
libraryResults.length
121-
} refreshed, ${libraryErrors.length} failed`,
122-
)
123-
124-
return {
125-
statusCode: 200,
126-
body: JSON.stringify({
127-
success: true,
128-
duration,
129-
stats: {
130-
npm: {
131-
totalDownloads: npmStats.totalDownloads,
132-
packageCount: Object.keys(npmStats.packageStats || {}).length,
133-
},
134-
github: {
135-
org: {
136-
starCount: githubStats.starCount,
137-
contributorCount: githubStats.contributorCount,
138-
dependentCount: githubStats.dependentCount,
139-
},
140-
libraries: {
141-
refreshed: libraryResults.length,
142-
failed: libraryErrors.length,
143-
results: libraryResults,
144-
errors: libraryErrors,
145-
},
146-
},
147-
},
148-
}),
149-
}
111+
} refreshed, ${libraryErrors.length} failed`
112+
);
113+
console.log(
114+
"[refresh-stats-cache-background] Next invocation at:",
115+
next_run
116+
);
150117
} catch (error) {
151-
const duration = Date.now() - startTime
152-
const errorMessage = error instanceof Error ? error.message : String(error)
153-
const errorStack = error instanceof Error ? error.stack : undefined
118+
const duration = Date.now() - startTime;
119+
const errorMessage = error instanceof Error ? error.message : String(error);
120+
const errorStack = error instanceof Error ? error.stack : undefined;
154121

155122
console.error(
156123
`[refresh-stats-cache-background] ✗ Failed after ${duration}ms:`,
157-
errorMessage,
158-
)
124+
errorMessage
125+
);
159126
if (errorStack) {
160-
console.error('[refresh-stats-cache-background] Stack:', errorStack)
161-
}
162-
163-
return {
164-
statusCode: 500,
165-
body: JSON.stringify({
166-
success: false,
167-
duration,
168-
error: errorMessage,
169-
stack: errorStack,
170-
}),
127+
console.error("[refresh-stats-cache-background] Stack:", errorStack);
171128
}
172129
}
173-
}
130+
};
174131

175-
/**
176-
* Netlify function configuration
177-
* - type: 'experimental-background' enables background execution (15 min timeout, returns 202 immediately)
178-
* - schedule: Cron expression for scheduled execution (runs every 6 hours)
179-
*/
180-
export const config = {
181-
type: 'experimental-background' as const,
182-
schedule: '0 */6 * * *', // Every 6 hours
183-
}
132+
export default handler;
133+
134+
export const config: Config = {
135+
schedule: "0 */6 * * *", // Every 6 hours
136+
};

‎netlify/functions/sync-github-releases.ts‎

Lines changed: 30 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -1,86 +1,53 @@
1-
import type { Handler, HandlerEvent, HandlerContext } from '@netlify/functions'
2-
import { syncGitHubReleases } from '~/server/feed/github.functions'
1+
import type { Config } from "@netlify/functions";
2+
import { syncGitHubReleases } from "~/server/feed/github.functions";
33

44
/**
5-
* Netlify Background + Scheduled Function - Sync GitHub releases
6-
*
7-
* This function combines both background and scheduled execution:
8-
* - Can be invoked via HTTP POST (returns 202 immediately, runs in background)
9-
* - Can be triggered on a schedule (runs automatically via cron)
10-
*
11-
* Background functions have a 15-minute timeout (vs 30 seconds for regular functions)
12-
* and return a 202 response immediately while processing continues in the background.
5+
* Netlify Scheduled Function - Sync GitHub releases
136
*
147
* This function syncs GitHub releases from all TanStack repos:
158
* - Fetches new releases since last sync
169
* - Creates/updates feed items in the database
1710
* - Marks releases as synced to avoid duplicates
1811
*
19-
* Scheduled: Runs automatically every 6 hours (configured via export config)
20-
* Only called by Netlify's scheduler - not publicly accessible
21-
* Timeout: 15 minutes
12+
* Scheduled: Runs automatically every 6 hours
2213
*/
23-
export const handler: Handler = async (
24-
event: HandlerEvent,
25-
context: HandlerContext,
26-
) => {
14+
const handler = async (req: Request) => {
15+
const { next_run } = await req.json();
16+
2717
console.log(
28-
'[sync-github-releases-background] Starting GitHub release sync...',
29-
)
18+
"[sync-github-releases-background] Starting GitHub release sync..."
19+
);
3020

31-
const startTime = Date.now()
21+
const startTime = Date.now();
3222

3323
try {
34-
const result = await syncGitHubReleases()
24+
const result = await syncGitHubReleases();
3525

36-
const duration = Date.now() - startTime
26+
const duration = Date.now() - startTime;
3727
console.log(
38-
`[sync-github-releases-background] ✓ Completed in ${duration}ms - Synced: ${result.syncedCount}, Skipped: ${result.skippedCount}, Errors: ${result.errorCount}`,
39-
)
40-
41-
return {
42-
statusCode: 200,
43-
body: JSON.stringify({
44-
success: true,
45-
duration,
46-
result: {
47-
syncedCount: result.syncedCount,
48-
skippedCount: result.skippedCount,
49-
errorCount: result.errorCount,
50-
},
51-
}),
52-
}
28+
`[sync-github-releases-background] ✓ Completed in ${duration}ms - Synced: ${result.syncedCount}, Skipped: ${result.skippedCount}, Errors: ${result.errorCount}`
29+
);
30+
console.log(
31+
"[sync-github-releases-background] Next invocation at:",
32+
next_run
33+
);
5334
} catch (error) {
54-
const duration = Date.now() - startTime
55-
const errorMessage = error instanceof Error ? error.message : String(error)
56-
const errorStack = error instanceof Error ? error.stack : undefined
35+
const duration = Date.now() - startTime;
36+
const errorMessage = error instanceof Error ? error.message : String(error);
37+
const errorStack = error instanceof Error ? error.stack : undefined;
5738

5839
console.error(
5940
`[sync-github-releases-background] ✗ Failed after ${duration}ms:`,
60-
errorMessage,
61-
)
41+
errorMessage
42+
);
6243
if (errorStack) {
63-
console.error('[sync-github-releases-background] Stack:', errorStack)
64-
}
65-
66-
return {
67-
statusCode: 500,
68-
body: JSON.stringify({
69-
success: false,
70-
duration,
71-
error: errorMessage,
72-
stack: errorStack,
73-
}),
44+
console.error("[sync-github-releases-background] Stack:", errorStack);
7445
}
7546
}
76-
}
47+
};
7748

78-
/**
79-
* Netlify function configuration
80-
* - type: 'experimental-background' enables background execution (15 min timeout, returns 202 immediately)
81-
* - schedule: Cron expression for scheduled execution (runs every 6 hours)
82-
*/
83-
export const config = {
84-
type: 'experimental-background' as const,
85-
schedule: '0 */6 * * *', // Every 6 hours
86-
}
49+
export default handler;
50+
51+
export const config: Config = {
52+
schedule: "0 */6 * * *", // Every 6 hours
53+
};

0 commit comments

Comments
 (0)