Skip to content

Commit 640a655

Browse files
LadyBluenotesLadyBluenotes
committed
refactor: share intent package discovery across admin and background sync
1 parent 5757ade commit 640a655

File tree

2 files changed

+187
-159
lines changed

2 files changed

+187
-159
lines changed

‎src/utils/intent-admin.server.ts‎

Lines changed: 6 additions & 159 deletions
Original file line numberDiff line numberDiff line change
@@ -14,17 +14,18 @@ import {
1414
import { eq, desc, sql } from 'drizzle-orm'
1515
import { requireCapability } from './auth.server'
1616
import {
17-
searchIntentPackages,
1817
fetchPackument,
19-
isIntentCompatible,
2018
selectVersionsToSync,
2119
extractSkillsFromTarball,
2220
} from './intent.server'
21+
import {
22+
discoverIntentPackagesFromNpm,
23+
discoverIntentPackagesViaGitHub,
24+
} from './intent-discovery.server'
2325
import {
2426
upsertIntentPackage,
2527
getKnownVersions,
2628
enqueuePackageVersion,
27-
markPackageVerified,
2829
replaceSkillsForVersion,
2930
getPendingVersions,
3031
markVersionSynced,
@@ -154,48 +155,7 @@ export const triggerIntentDiscover = createServerFn({
154155
}).handler(async () => {
155156
await requireCapability({ data: { capability: 'admin' } })
156157

157-
let packagesDiscovered = 0
158-
let packagesVerified = 0
159-
let versionsEnqueued = 0
160-
const errors: Array<string> = []
161-
162-
const searchResults = await searchIntentPackages()
163-
packagesDiscovered = searchResults.objects.length
164-
165-
for (const { package: pkg } of searchResults.objects) {
166-
try {
167-
await upsertIntentPackage({ name: pkg.name, verified: false })
168-
169-
const packument = await fetchPackument(pkg.name)
170-
const latestVersion = packument['dist-tags'].latest
171-
if (!latestVersion) continue
172-
173-
const latestMeta = packument.versions[latestVersion]
174-
if (!latestMeta || !isIntentCompatible(latestMeta)) continue
175-
176-
await markPackageVerified(pkg.name)
177-
packagesVerified++
178-
179-
const knownVersions = await getKnownVersions(pkg.name)
180-
const versionsToEnqueue = selectVersionsToSync(packument, knownVersions)
181-
182-
for (const { version, tarball, publishedAt } of versionsToEnqueue) {
183-
await enqueuePackageVersion({
184-
packageName: pkg.name,
185-
version,
186-
tarballUrl: tarball,
187-
publishedAt,
188-
})
189-
versionsEnqueued++
190-
}
191-
} catch (err) {
192-
errors.push(
193-
`${pkg.name}: ${err instanceof Error ? err.message : String(err)}`,
194-
)
195-
}
196-
}
197-
198-
return { packagesDiscovered, packagesVerified, versionsEnqueued, errors }
158+
return discoverIntentPackagesFromNpm()
199159
})
200160

201161
// ---------------------------------------------------------------------------
@@ -294,120 +254,7 @@ export const discoverViaGitHub = createServerFn({ method: 'POST' }).handler(
294254
async () => {
295255
await requireCapability({ data: { capability: 'admin' } })
296256

297-
const token = process.env.GITHUB_AUTH_TOKEN
298-
if (!token) throw new Error('GITHUB_AUTH_TOKEN not set')
299-
300-
const ghHeaders = {
301-
Authorization: `Bearer ${token}`,
302-
Accept: 'application/vnd.github.v3+json',
303-
}
304-
305-
// Search GitHub for package.json files referencing @tanstack/intent
306-
const searchRes = await fetch(
307-
'https://api.github.com/search/code?q=%22%40tanstack%2Fintent%22+filename%3Apackage.json&per_page=100',
308-
{ headers: ghHeaders },
309-
)
310-
if (!searchRes.ok)
311-
throw new Error(`GitHub search failed: ${searchRes.status}`)
312-
const searchData = (await searchRes.json()) as {
313-
total_count: number
314-
items: Array<{
315-
path: string
316-
repository: { full_name: string }
317-
url: string
318-
}>
319-
}
320-
321-
// Deduplicate: one package.json per repo+path
322-
const seen = new Set<string>()
323-
const candidates: Array<{ repo: string; path: string }> = []
324-
for (const item of searchData.items) {
325-
const key = `${item.repository.full_name}|${item.path}`
326-
if (!seen.has(key)) {
327-
seen.add(key)
328-
candidates.push({ repo: item.repository.full_name, path: item.path })
329-
}
330-
}
331-
332-
const results = {
333-
searched: candidates.length,
334-
checkedOnNpm: 0,
335-
hadSkills: 0,
336-
enqueued: 0,
337-
skipped: 0,
338-
errors: [] as Array<string>,
339-
}
340-
341-
for (const { repo, path } of candidates) {
342-
try {
343-
// Fetch the package.json content from GitHub
344-
const contentRes = await fetch(
345-
`https://api.github.com/repos/${repo}/contents/${path}`,
346-
{ headers: ghHeaders },
347-
)
348-
if (!contentRes.ok) continue
349-
const contentData = (await contentRes.json()) as {
350-
content?: string
351-
encoding?: string
352-
}
353-
if (!contentData.content) continue
354-
355-
const pkgJson = JSON.parse(
356-
Buffer.from(contentData.content, 'base64').toString('utf-8'),
357-
) as { name?: string; private?: boolean }
358-
359-
const pkgName = pkgJson.name
360-
if (!pkgName || pkgJson.private) continue
361-
362-
results.checkedOnNpm++
363-
364-
// Check NPM for the package and its latest tarball
365-
const npmRes = await fetch(
366-
`https://registry.npmjs.org/${encodeURIComponent(pkgName)}/latest`,
367-
)
368-
if (!npmRes.ok) continue // not published
369-
370-
const npmMeta = (await npmRes.json()) as {
371-
version?: string
372-
dist?: { tarball?: string }
373-
}
374-
const version = npmMeta.version
375-
const tarballUrl = npmMeta.dist?.tarball
376-
if (!version || !tarballUrl) continue
377-
378-
// Peek at tarball for SKILL.md files (stream headers only)
379-
const skills = await extractSkillsFromTarball(tarballUrl)
380-
if (skills.length === 0) {
381-
results.skipped++
382-
continue
383-
}
384-
385-
results.hadSkills++
386-
387-
// Seed into DB
388-
await upsertIntentPackage({ name: pkgName, verified: true })
389-
390-
const knownVersions = await getKnownVersions(pkgName)
391-
const packument = await fetchPackument(pkgName)
392-
const versionsToEnqueue = selectVersionsToSync(packument, knownVersions)
393-
394-
for (const v of versionsToEnqueue) {
395-
await enqueuePackageVersion({
396-
packageName: pkgName,
397-
version: v.version,
398-
tarballUrl: v.tarball,
399-
publishedAt: v.publishedAt,
400-
})
401-
results.enqueued++
402-
}
403-
} catch (err) {
404-
results.errors.push(
405-
`${repo}/${path}: ${err instanceof Error ? err.message : String(err)}`,
406-
)
407-
}
408-
}
409-
410-
return results
257+
return discoverIntentPackagesViaGitHub()
411258
},
412259
)
413260

‎src/utils/intent-discovery.server.ts‎

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
import {
2+
searchIntentPackages,
3+
fetchPackument,
4+
isIntentCompatible,
5+
selectVersionsToSync,
6+
extractSkillsFromTarball,
7+
} from './intent.server'
8+
import {
9+
upsertIntentPackage,
10+
getKnownVersions,
11+
enqueuePackageVersion,
12+
markPackageVerified,
13+
} from './intent-db.server'
14+
15+
export type IntentNpmDiscoveryResult = {
16+
packagesDiscovered: number
17+
packagesVerified: number
18+
versionsEnqueued: number
19+
errors: Array<string>
20+
}
21+
22+
export type IntentGitHubDiscoveryResult = {
23+
searched: number
24+
checkedOnNpm: number
25+
hadSkills: number
26+
enqueued: number
27+
skipped: number
28+
errors: Array<string>
29+
}
30+
31+
type NpmLatestMetadata = {
32+
version?: string
33+
dist?: { tarball?: string }
34+
}
35+
36+
async function enqueueDiscoveredVersions(packageName: string) {
37+
const packument = await fetchPackument(packageName)
38+
const knownVersions = await getKnownVersions(packageName)
39+
const versionsToEnqueue = selectVersionsToSync(packument, knownVersions)
40+
41+
for (const { version, tarball, publishedAt } of versionsToEnqueue) {
42+
await enqueuePackageVersion({
43+
packageName,
44+
version,
45+
tarballUrl: tarball,
46+
publishedAt,
47+
})
48+
}
49+
50+
return versionsToEnqueue.length
51+
}
52+
53+
export async function discoverIntentPackagesFromNpm(): Promise<IntentNpmDiscoveryResult> {
54+
let packagesDiscovered = 0
55+
let packagesVerified = 0
56+
let versionsEnqueued = 0
57+
const errors: Array<string> = []
58+
59+
const searchResults = await searchIntentPackages()
60+
packagesDiscovered = searchResults.objects.length
61+
62+
for (const { package: pkg } of searchResults.objects) {
63+
try {
64+
await upsertIntentPackage({ name: pkg.name, verified: false })
65+
66+
const packument = await fetchPackument(pkg.name)
67+
const latestVersion = packument['dist-tags'].latest
68+
if (!latestVersion) continue
69+
70+
const latestMeta = packument.versions[latestVersion]
71+
if (!latestMeta || !isIntentCompatible(latestMeta)) continue
72+
73+
await markPackageVerified(pkg.name)
74+
packagesVerified++
75+
76+
versionsEnqueued += await enqueueDiscoveredVersions(pkg.name)
77+
} catch (error) {
78+
errors.push(
79+
`${pkg.name}: ${error instanceof Error ? error.message : String(error)}`,
80+
)
81+
}
82+
}
83+
84+
return { packagesDiscovered, packagesVerified, versionsEnqueued, errors }
85+
}
86+
87+
export async function discoverIntentPackagesViaGitHub(): Promise<IntentGitHubDiscoveryResult> {
88+
const githubToken = process.env.GITHUB_AUTH_TOKEN
89+
if (!githubToken) throw new Error('GITHUB_AUTH_TOKEN not set')
90+
91+
const ghHeaders = {
92+
Authorization: `Bearer ${githubToken}`,
93+
Accept: 'application/vnd.github.v3+json',
94+
}
95+
96+
const searchRes = await fetch(
97+
'https://api.github.com/search/code?q=%22%40tanstack%2Fintent%22+filename%3Apackage.json&per_page=100',
98+
{ headers: ghHeaders },
99+
)
100+
if (!searchRes.ok) {
101+
throw new Error(`GitHub search failed: ${searchRes.status}`)
102+
}
103+
104+
const searchData = (await searchRes.json()) as {
105+
items: Array<{ path: string; repository: { full_name: string } }>
106+
}
107+
108+
const seenRepoPaths = new Set<string>()
109+
const candidates = searchData.items.filter((item) => {
110+
const key = `${item.repository.full_name}|${item.path}`
111+
if (seenRepoPaths.has(key)) return false
112+
seenRepoPaths.add(key)
113+
return true
114+
})
115+
116+
const results: IntentGitHubDiscoveryResult = {
117+
searched: candidates.length,
118+
checkedOnNpm: 0,
119+
hadSkills: 0,
120+
enqueued: 0,
121+
skipped: 0,
122+
errors: [],
123+
}
124+
125+
const seenPackageNames = new Set<string>()
126+
127+
for (const { repo, path } of candidates.map((item) => ({
128+
repo: item.repository.full_name,
129+
path: item.path,
130+
}))) {
131+
try {
132+
const contentRes = await fetch(
133+
`https://api.github.com/repos/${repo}/contents/${path}`,
134+
{ headers: ghHeaders },
135+
)
136+
if (!contentRes.ok) continue
137+
138+
const contentData = (await contentRes.json()) as { content?: string }
139+
if (!contentData.content) continue
140+
141+
const pkgJson = JSON.parse(
142+
Buffer.from(contentData.content, 'base64').toString('utf-8'),
143+
) as { name?: string; private?: boolean }
144+
145+
const pkgName = pkgJson.name
146+
if (!pkgName || pkgJson.private || seenPackageNames.has(pkgName)) continue
147+
seenPackageNames.add(pkgName)
148+
149+
results.checkedOnNpm++
150+
151+
const npmRes = await fetch(
152+
`https://registry.npmjs.org/${encodeURIComponent(pkgName)}/latest`,
153+
)
154+
if (!npmRes.ok) continue
155+
156+
const npmMeta = (await npmRes.json()) as NpmLatestMetadata
157+
const version = npmMeta.version
158+
const tarballUrl = npmMeta.dist?.tarball
159+
if (!version || !tarballUrl) continue
160+
161+
const skills = await extractSkillsFromTarball(tarballUrl)
162+
if (skills.length === 0) {
163+
results.skipped++
164+
continue
165+
}
166+
167+
results.hadSkills++
168+
169+
await upsertIntentPackage({ name: pkgName, verified: true })
170+
await markPackageVerified(pkgName)
171+
172+
results.enqueued += await enqueueDiscoveredVersions(pkgName)
173+
} catch (error) {
174+
results.errors.push(
175+
`${repo}/${path}: ${error instanceof Error ? error.message : String(error)}`,
176+
)
177+
}
178+
}
179+
180+
return results
181+
}

0 commit comments

Comments
 (0)