Skip to content

Commit c5f647d

Browse files
LadyBluenotesLadyBluenotes
committed
perf: reduce github fan-out in example deploy and name checks
1 parent 91f76c2 commit c5f647d

File tree

4 files changed

+275
-227
lines changed

4 files changed

+275
-227
lines changed

‎netlify/functions/sync-intent-discover-background.ts‎

Lines changed: 17 additions & 149 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,8 @@
11
import type { Config } from '@netlify/functions'
22
import {
3-
searchIntentPackages,
4-
fetchPackument,
5-
isIntentCompatible,
6-
selectVersionsToSync,
7-
extractSkillsFromTarball,
8-
} from '~/utils/intent.server'
9-
import {
10-
upsertIntentPackage,
11-
getKnownVersions,
12-
enqueuePackageVersion,
13-
markPackageVerified,
14-
} from '~/utils/intent-db.server'
3+
discoverIntentPackagesFromNpm,
4+
discoverIntentPackagesViaGitHub,
5+
} from '~/utils/intent-discovery.server'
156

167
/**
178
* Netlify Scheduled Function - Discover Intent-compatible npm packages
@@ -36,163 +27,40 @@ const handler = async (req: Request) => {
3627
let versionsEnqueued = 0
3728
const errors: Array<string> = []
3829

39-
// ---------------------------------------------------------------------------
40-
// Path 1: NPM keyword search
41-
// ---------------------------------------------------------------------------
4230
try {
4331
console.log(
4432
'[intent-discover] Searching NPM for keywords:tanstack-intent...',
4533
)
46-
const searchResults = await searchIntentPackages()
34+
const npmResult = await discoverIntentPackagesFromNpm()
35+
versionsEnqueued += npmResult.versionsEnqueued
36+
errors.push(...npmResult.errors.map((error) => `npm/${error}`))
37+
4738
console.log(
48-
`[intent-discover] NPM found ${searchResults.objects.length} candidates`,
39+
`[intent-discover] NPM found ${npmResult.packagesDiscovered} candidates and verified ${npmResult.packagesVerified}`,
4940
)
50-
51-
for (const { package: pkg } of searchResults.objects) {
52-
try {
53-
await upsertIntentPackage({ name: pkg.name, verified: false })
54-
55-
const packument = await fetchPackument(pkg.name)
56-
const latestVersion = packument['dist-tags'].latest
57-
if (!latestVersion) continue
58-
59-
const latestMeta = packument.versions[latestVersion]
60-
if (!latestMeta || !isIntentCompatible(latestMeta)) continue
61-
62-
await markPackageVerified(pkg.name)
63-
64-
const knownVersions = await getKnownVersions(pkg.name)
65-
const toEnqueue = selectVersionsToSync(packument, knownVersions)
66-
for (const { version, tarball, publishedAt } of toEnqueue) {
67-
await enqueuePackageVersion({
68-
packageName: pkg.name,
69-
version,
70-
tarballUrl: tarball,
71-
publishedAt,
72-
})
73-
versionsEnqueued++
74-
}
75-
console.log(
76-
`[intent-discover] NPM: ${pkg.name} - enqueued ${toEnqueue.length}`,
77-
)
78-
} catch (e) {
79-
const msg = `npm/${pkg.name}: ${e instanceof Error ? e.message : String(e)}`
80-
console.error(`[intent-discover] ${msg}`)
81-
errors.push(msg)
82-
}
83-
}
84-
} catch (e) {
41+
} catch (error) {
8542
console.error(
8643
'[intent-discover] NPM path failed:',
87-
e instanceof Error ? e.message : String(e),
44+
error instanceof Error ? error.message : String(error),
8845
)
8946
}
9047

91-
// ---------------------------------------------------------------------------
92-
// Path 2: GitHub code search for @tanstack/intent dependents
93-
// ---------------------------------------------------------------------------
94-
const githubToken = process.env.GITHUB_AUTH_TOKEN
95-
if (githubToken) {
48+
if (process.env.GITHUB_AUTH_TOKEN) {
9649
try {
9750
console.log(
9851
'[intent-discover] Searching GitHub for @tanstack/intent dependents...',
9952
)
100-
const ghHeaders = {
101-
Authorization: `Bearer ${githubToken}`,
102-
Accept: 'application/vnd.github.v3+json',
103-
}
104-
105-
const searchRes = await fetch(
106-
'https://api.github.com/search/code?q=%22%40tanstack%2Fintent%22+filename%3Apackage.json&per_page=100',
107-
{ headers: ghHeaders },
108-
)
109-
if (!searchRes.ok) throw new Error(`GitHub search ${searchRes.status}`)
110-
111-
const searchData = (await searchRes.json()) as {
112-
items: Array<{ path: string; repository: { full_name: string } }>
113-
}
114-
115-
// Deduplicate repo+path pairs
116-
const seen = new Set<string>()
117-
const candidates = searchData.items.filter((item) => {
118-
const key = `${item.repository.full_name}|${item.path}`
119-
if (seen.has(key)) return false
120-
seen.add(key)
121-
return true
122-
})
53+
const githubResult = await discoverIntentPackagesViaGitHub()
54+
versionsEnqueued += githubResult.enqueued
55+
errors.push(...githubResult.errors.map((error) => `github/${error}`))
12356

12457
console.log(
125-
`[intent-discover] GitHub found ${candidates.length} package.json files`,
58+
`[intent-discover] GitHub searched ${githubResult.searched} candidates, checked ${githubResult.checkedOnNpm} packages, found ${githubResult.hadSkills} with skills`,
12659
)
127-
128-
for (const { repo, path } of candidates.map((i) => ({
129-
repo: i.repository.full_name,
130-
path: i.path,
131-
}))) {
132-
try {
133-
const contentRes = await fetch(
134-
`https://api.github.com/repos/${repo}/contents/${path}`,
135-
{ headers: ghHeaders },
136-
)
137-
if (!contentRes.ok) continue
138-
139-
const contentData = (await contentRes.json()) as { content?: string }
140-
if (!contentData.content) continue
141-
142-
const pkgJson = JSON.parse(
143-
Buffer.from(contentData.content, 'base64').toString('utf-8'),
144-
) as { name?: string; private?: boolean }
145-
146-
const pkgName = pkgJson.name
147-
if (!pkgName || pkgJson.private) continue
148-
149-
// Check NPM
150-
const npmRes = await fetch(
151-
`https://registry.npmjs.org/${encodeURIComponent(pkgName)}/latest`,
152-
)
153-
if (!npmRes.ok) continue
154-
155-
const npmMeta = (await npmRes.json()) as {
156-
version?: string
157-
dist?: { tarball?: string }
158-
}
159-
if (!npmMeta.version || !npmMeta.dist?.tarball) continue
160-
161-
// Peek at tarball for skills
162-
const skills = await extractSkillsFromTarball(npmMeta.dist.tarball)
163-
if (skills.length === 0) continue
164-
165-
await upsertIntentPackage({ name: pkgName, verified: true })
166-
await markPackageVerified(pkgName)
167-
168-
const packument = await fetchPackument(pkgName)
169-
const knownVersions = await getKnownVersions(pkgName)
170-
const toEnqueue = selectVersionsToSync(packument, knownVersions)
171-
172-
for (const { version, tarball, publishedAt } of toEnqueue) {
173-
await enqueuePackageVersion({
174-
packageName: pkgName,
175-
version,
176-
tarballUrl: tarball,
177-
publishedAt,
178-
})
179-
versionsEnqueued++
180-
}
181-
if (toEnqueue.length > 0) {
182-
console.log(
183-
`[intent-discover] GitHub: ${pkgName} - enqueued ${toEnqueue.length}`,
184-
)
185-
}
186-
} catch (e) {
187-
const msg = `github/${repo}: ${e instanceof Error ? e.message : String(e)}`
188-
console.error(`[intent-discover] ${msg}`)
189-
errors.push(msg)
190-
}
191-
}
192-
} catch (e) {
60+
} catch (error) {
19361
console.error(
19462
'[intent-discover] GitHub path failed:',
195-
e instanceof Error ? e.message : String(e),
63+
error instanceof Error ? error.message : String(error),
19664
)
19765
}
19866
} else {

‎src/routes/api/builder/deploy/check-name.ts‎

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
import { createFileRoute } from '@tanstack/react-router'
22
import { getAuthService } from '~/auth/index.server'
33
import { getGitHubAuthState } from '~/auth/github.server'
4-
import { checkRepoNameAvailable } from '~/utils/github-repo.server'
4+
import {
5+
checkRepoNameAvailable,
6+
validateRepoName,
7+
} from '~/utils/github-repo.server'
58

69
export const Route = createFileRoute('/api/builder/deploy/check-name' as any)({
710
server: {
@@ -11,27 +14,46 @@ export const Route = createFileRoute('/api/builder/deploy/check-name' as any)({
1114
const name = url.searchParams.get('name')
1215

1316
if (!name) {
14-
return Response.json({ available: false, error: 'Name required' }, { status: 400 })
17+
return Response.json(
18+
{ available: false, error: 'Name required' },
19+
{ status: 400 },
20+
)
21+
}
22+
23+
const validation = validateRepoName(name)
24+
if (!validation.valid) {
25+
return Response.json({
26+
available: false,
27+
error: validation.error ?? 'Invalid repository name',
28+
})
1529
}
1630

1731
const authService = getAuthService()
1832
const user = await authService.getCurrentUser(request)
1933

2034
if (!user) {
21-
return Response.json({ available: false, error: 'Not authenticated' }, { status: 401 })
35+
return Response.json(
36+
{ available: false, error: 'Not authenticated' },
37+
{ status: 401 },
38+
)
2239
}
2340

2441
const authState = await getGitHubAuthState(user.userId)
2542

2643
if (!authState.hasRepoScope || !authState.accessToken) {
27-
return Response.json({ available: false, error: 'Missing scope' }, { status: 403 })
44+
return Response.json(
45+
{ available: false, error: 'Missing scope' },
46+
{ status: 403 },
47+
)
2848
}
2949

3050
try {
31-
const available = await checkRepoNameAvailable(authState.accessToken, name)
51+
const available = await checkRepoNameAvailable(
52+
authState.accessToken,
53+
name,
54+
)
3255
return Response.json({ available })
3356
} catch {
34-
// On error, assume available (don't block the user)
3557
return Response.json({ available: true })
3658
}
3759
},

0 commit comments

Comments
 (0)