diff --git a/backend/src/osspckgs/migrations/V1782978597__package_maintainers_ingestion_source.sql b/backend/src/osspckgs/migrations/V1782978597__package_maintainers_ingestion_source.sql new file mode 100644 index 0000000000..7f2cf53d51 --- /dev/null +++ b/backend/src/osspckgs/migrations/V1782978597__package_maintainers_ingestion_source.sql @@ -0,0 +1,2 @@ +ALTER TABLE package_maintainers + ADD COLUMN IF NOT EXISTS ingestion_source text; diff --git a/services/apps/packages_worker/package.json b/services/apps/packages_worker/package.json index f8d5981650..e48f6a3ec9 100644 --- a/services/apps/packages_worker/package.json +++ b/services/apps/packages_worker/package.json @@ -51,6 +51,8 @@ "dev:nuget-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=nuget-worker SERVICE=nuget-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9242 src/bin/nuget-worker.ts", "backfill:maven": "SERVICE=maven tsx src/bin/maven-backfill.ts", "backfill:maven:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=maven LOG_LEVEL=info tsx src/bin/maven-backfill.ts", + "import:maven-maintainers": "SERVICE=maven tsx src/maven/scripts/importMaintainersFromCsv.ts", + "import:maven-maintainers:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=maven tsx src/maven/scripts/importMaintainersFromCsv.ts", "backfill:stewardship": "SERVICE=stewardship-backfill tsx src/bin/stewardship-backfill.ts", "backfill:stewardship:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=stewardship-backfill LOG_LEVEL=info tsx src/bin/stewardship-backfill.ts", "monitor:osspckgs": "SERVICE=bq-dataset-ingest tsx src/scripts/monitorOsspckgs.ts", diff --git a/services/apps/packages_worker/src/maven/scripts/csv.ts b/services/apps/packages_worker/src/maven/scripts/csv.ts new file mode 100644 index 0000000000..494d8997f7 --- /dev/null +++ b/services/apps/packages_worker/src/maven/scripts/csv.ts @@ -0,0 +1,56 @@ +export function parseCsv(content: string): Record[] { + const rows: string[][] = [] + let cur = '' + let inQuote = false + let row: string[] = [] + + for (let i = 0; i < content.length; i++) { + const ch = content[i] + const next = content[i + 1] + + if (inQuote) { + if (ch === '"' && next === '"') { + cur += '"' + i++ + } else if (ch === '"') { + inQuote = false + } else { + cur += ch + } + } else { + if (ch === '"') { + inQuote = true + } else if (ch === ',') { + row.push(cur) + cur = '' + } else if (ch === '\r' && next === '\n') { + row.push(cur) + cur = '' + rows.push(row) + row = [] + i++ + } else if (ch === '\n') { + row.push(cur) + cur = '' + rows.push(row) + row = [] + } else { + cur += ch + } + } + } + if (cur || row.length) { + row.push(cur) + rows.push(row) + } + + if (rows.length === 0) return [] + const headers = rows[0] + return rows.slice(1).map((r) => { + const obj: Record = {} + headers.forEach((h, i) => { + obj[h.trim()] = (r[i] ?? '').trim() + }) + return obj + }) +} diff --git a/services/apps/packages_worker/src/maven/scripts/fetchMavenMaintainers.ts b/services/apps/packages_worker/src/maven/scripts/fetchMavenMaintainers.ts new file mode 100644 index 0000000000..a6a36d1fa3 --- /dev/null +++ b/services/apps/packages_worker/src/maven/scripts/fetchMavenMaintainers.ts @@ -0,0 +1,675 @@ +/** + * Fetches maintainer data for Maven packages from Libraries.io and GitHub. + * + * For each package in the input CSV it: + * 1. Calls Libraries.io to get the linked GitHub repo URL + * 2. Looks for CODEOWNERS or MAINTAINERS.md in the repo + * 3. Falls back to top GitHub contributors if neither file is found + * + * Output: CSV with one row per (package, maintainer) ready for manual review. + * Packages with no maintainer found still get an empty row so nothing is lost. + * + * Env vars: + * LIBRARIES_IO_API_KEY – https://libraries.io/api_key + * GITHUB_TOKEN – PAT with public_repo scope + * + * Usage: + * LIBRARIES_IO_API_KEY=xxx GITHUB_TOKEN=yyy \ + * tsx src/maven/scripts/fetchMavenMaintainers.ts /path/to/packages_top500.csv [output.csv] + */ +import axios, { AxiosInstance } from 'axios' +import * as fs from 'fs' +import * as path from 'path' + +import { extractArtifact, normalizeScmUrl } from '../extract' +import { resolveLatestVersion } from '../metadata' +import { resolveRegistryBaseUrl } from '../registry' + +import { parseCsv } from './csv' + +// ─── Config ─────────────────────────────────────────────────────────────────── + +const LIBRARIES_IO_KEY = process.env.LIBRARIES_IO_API_KEY ?? '' +const GITHUB_TOKEN = process.env.GITHUB_TOKEN ?? '' + +// Libraries.io free tier: 60 req/min → 1 req/sec to stay safe +const LIBRARIES_IO_DELAY_MS = 1_100 +// GitHub: 5000 req/hr with token, small courtesy delay between batches +const GITHUB_DELAY_MS = 200 + +// Set to 0 for unlimited (paginates all pages) +const TOP_CONTRIBUTORS_LIMIT = 0 + +// ─── HTTP clients ───────────────────────────────────────────────────────────── + +const libIo: AxiosInstance = axios.create({ + baseURL: 'https://libraries.io/api', + timeout: 15_000, +}) + +const github: AxiosInstance = axios.create({ + baseURL: 'https://api.github.com', + timeout: 15_000, + headers: { + Authorization: GITHUB_TOKEN ? `Bearer ${GITHUB_TOKEN}` : undefined, + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28', + }, +}) + +// ─── Types ──────────────────────────────────────────────────────────────────── + +interface MaintainerRow { + package_purl: string + package_namespace: string + package_name: string + github_repo: string + repo_source: string + maintainer_github_login: string + maintainer_display_name: string + maintainer_email: string + maintainer_url: string + role: string + source: string + contributions: string + notes: string +} + +// ─── Helpers ────────────────────────────────────────────────────────────────── + +function sleep(ms: number): Promise { + return new Promise((r) => setTimeout(r, ms)) +} + +async function safeGet( + client: AxiosInstance, + url: string, + params?: Record, +): Promise { + const MAX_RETRIES = 5 + for (let attempt = 0; attempt < MAX_RETRIES; attempt++) { + try { + const res = await client.get(url, { params }) + return res.data + } catch (err) { + if (axios.isAxiosError(err)) { + const status = err.response?.status + if (status === 404 || status === 422) return null + if (status === 403 || status === 429) { + const reset = err.response?.headers?.['x-ratelimit-reset'] + const wait = reset ? Math.max(0, Number(reset) * 1000 - Date.now()) + 1000 : 60_000 + console.warn( + ` Rate limited (attempt ${attempt + 1}/${MAX_RETRIES}) — waiting ${Math.round(wait / 1000)}s`, + ) + await sleep(wait) + continue + } + console.warn(` HTTP ${status} for ${url}`) + } + return null + } + } + console.warn(` Giving up after ${MAX_RETRIES} rate-limit retries: ${url}`) + return null +} + +function csvEscape(value: string): string { + const s = String(value ?? '') + if (s.includes(',') || s.includes('"') || s.includes('\n')) { + return `"${s.replace(/"/g, '""')}"` + } + return s +} + +function toCsvLine(row: MaintainerRow): string { + return [ + row.package_purl, + row.package_namespace, + row.package_name, + row.github_repo, + row.repo_source, + row.maintainer_github_login, + row.maintainer_display_name, + row.maintainer_email, + row.maintainer_url, + row.role, + row.source, + row.contributions, + row.notes, + ] + .map(csvEscape) + .join(',') +} + +// ─── Libraries.io ───────────────────────────────────────────────────────────── + +interface LibIoPackage { + repository_url?: string + homepage?: string +} + +async function getLibIoRepoUrl(groupId: string, artifactId: string): Promise { + if (!LIBRARIES_IO_KEY) return null + const data = await safeGet( + libIo, + `/maven/${encodeURIComponent(`${groupId}:${artifactId}`)}`, + { api_key: LIBRARIES_IO_KEY }, + ) + return data?.repository_url ?? null +} + +// ─── POM SCM fallback ───────────────────────────────────────────────────────── + +async function getPomScmUrl(groupId: string, artifactId: string): Promise { + const baseUrl = resolveRegistryBaseUrl(groupId) + const version = await resolveLatestVersion(groupId, artifactId, baseUrl) + if (!version) return null + const extracted = await extractArtifact(groupId, artifactId, version, baseUrl) + return normalizeScmUrl(extracted.scmUrl) +} + +// ─── GitHub: parse owner/repo from URL ─────────────────────────────────────── + +/** + * Converts known non-GitHub SCM URLs to their GitHub mirror equivalent. + * + * Apache projects publish to gitbox.apache.org (and historically git.apache.org) + * but mirror everything to github.com/apache/. Eclipse similarly mirrors to + * github.com/eclipse/. This lets us use the GitHub API for those repos. + */ +function toGithubUrl(url: string): string { + // Apache Gitbox: https://gitbox.apache.org/repos/asf/ant.git → https://github.com/apache/ant + const apacheGitbox = url.match(/gitbox\.apache\.org\/repos\/asf\/([^/.]+?)(?:\.git)?(?:\/.*)?$/) + if (apacheGitbox) return `https://github.com/apache/${apacheGitbox[1]}` + + // Apache old-git: https://git-wip-us.apache.org/repos/asf/freemarker.git → https://github.com/apache/freemarker + const apacheWip = url.match(/git-wip-us\.apache\.org\/repos\/asf\/([^/.]+?)(?:\.git)?(?:\/.*)?$/) + if (apacheWip) return `https://github.com/apache/${apacheWip[1]}` + + // Apache Git (legacy): https://git.apache.org/ant.git → https://github.com/apache/ant + const apacheGit = url.match(/(?:^|\/)git\.apache\.org\/([^/.]+?)(?:\.git)?(?:\/.*)?$/) + if (apacheGit) return `https://github.com/apache/${apacheGit[1]}` + + // Eclipse: https://git.eclipse.org/c/platform/eclipse.platform.git → https://github.com/eclipse/eclipse.platform + const eclipse = url.match(/git\.eclipse\.org\/c\/(?:[^/]+\/)?([^/.]+?)(?:\.git)?(?:\/.*)?$/) + if (eclipse) return `https://github.com/eclipse/${eclipse[1]}` + + return url +} + +/** + * Hardcoded overrides for groupIds where Libraries.io consistently returns the + * wrong repo (parent POMs, broken templates) or where the Apache heuristic + * would produce the wrong project name. + * + * Key: groupId prefix (exact match or startsWith) + * Value: GitHub owner/repo + */ +const REPO_OVERRIDES: Record = { + // Jersey moved from java.net to Eclipse Foundation + 'com.sun.jersey': 'eclipse-ee4j/jersey', + 'com.sun.jersey.contribs': 'eclipse-ee4j/jersey', + 'com.sun.jersey.jersey-test-framework': 'eclipse-ee4j/jersey', + // Apache Ivy lives under ant-ivy, not ivy + 'org.apache.ivy': 'apache/ant-ivy', + // Apache Axis 1.x + 'org.apache.axis': 'apache/axis1-java', + // Apache Commons bare groupId — use artifactId (handled in guessApacheGithubUrl) + // Dirigible has a broken URL template in Libraries.io + 'org.eclipse.dirigible': 'eclipse/dirigible', + // Hudson is effectively dead/archived but let's point to the right place + 'org.eclipse.hudson': 'hudson/hudson', + 'org.jvnet.hudson.main': 'hudson/hudson', +} + +/** + * Returns a hardcoded GitHub owner/repo for known problematic groupIds, + * or null if no override exists. + */ +function getRepoOverride(groupId: string): string | null { + // Exact match first + if (REPO_OVERRIDES[groupId]) return `https://github.com/${REPO_OVERRIDES[groupId]}` + // Prefix match + for (const prefix of Object.keys(REPO_OVERRIDES)) { + if (groupId.startsWith(prefix + '.')) return `https://github.com/${REPO_OVERRIDES[prefix]}` + } + return null +} + +/** + * Returns true for repo URLs that are known to be parent POMs or broken templates, + * not actual project repos worth querying. + */ +function isJunkRepoUrl(repoUrl: string): boolean { + return ( + repoUrl.includes('sonatype/jvnet-parent') || repoUrl.includes('${') // broken URL templates + ) +} + +/** + * Last-resort heuristic for org.apache.* packages when SCM URL extraction fails. + * Apache mirrors all its projects to github.com/apache/ using the project name. + * Derives the project name from the groupId sub-namespace (e.g. org.apache.kafka → kafka). + * Returns null for non-Apache groupIds. + */ +function guessApacheGithubUrl(groupId: string, artifactId: string): string | null { + if (!groupId.startsWith('org.apache') && !groupId.startsWith('commons-')) return null + + if (groupId.startsWith('org.apache.')) { + const sub = groupId.slice('org.apache.'.length) + const firstSegment = sub.split('.')[0] + + // org.apache.commons → commons projects follow apache/commons-{artifactId} convention + // e.g. org.apache.commons + commons-io → apache/commons-io + if (firstSegment === 'commons') { + return `https://github.com/apache/${artifactId}` + } + + // org.apache.tomcat.embed → tomcat (use first segment) + return `https://github.com/apache/${firstSegment}` + } + + if (groupId === 'org.apache') { + // Bare org.apache — use artifactId as-is + return `https://github.com/apache/${artifactId}` + } + + if (groupId.startsWith('commons-')) { + // commons-collections:commons-collections → apache/commons-collections + return `https://github.com/apache/${groupId}` + } + + return null +} + +function parseGithubRepo(url: string): { owner: string; repo: string } | null { + const resolved = toGithubUrl(url) + const m = resolved.match(/github\.com\/([^/]+)\/([^/]+?)(?:\.git)?(?:\/.*)?$/) + if (!m) return null + return { owner: m[1], repo: m[2] } +} + +// ─── GitHub: CODEOWNERS ─────────────────────────────────────────────────────── + +const CODEOWNERS_PATHS = ['CODEOWNERS', '.github/CODEOWNERS', 'docs/CODEOWNERS'] + +async function fetchCODEOWNERS(owner: string, repo: string): Promise { + for (const p of CODEOWNERS_PATHS) { + const data = await safeGet<{ content?: string }>( + github, + `/repos/${owner}/${repo}/contents/${p}`, + ) + if (data?.content) { + return Buffer.from(data.content, 'base64').toString('utf-8') + } + } + return null +} + +// Extract @username handles from CODEOWNERS (skip @org/team entries) +function parseCODEOWNERS(content: string): string[] { + const logins: string[] = [] + const seen = new Set() + for (const line of content.split('\n')) { + const trimmed = line.trim() + if (!trimmed || trimmed.startsWith('#')) continue + // Match @login — skip @org/team entries (slash immediately after the handle) + let match: RegExpExecArray | null + const re = /@([a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?)/g + while ((match = re.exec(trimmed)) !== null) { + if (trimmed[match.index + match[0].length] === '/') continue + const login = match[1].toLowerCase() + if (!seen.has(login)) { + seen.add(login) + logins.push(login) + } + } + } + return logins +} + +// ─── GitHub: MAINTAINERS file ───────────────────────────────────────────────── + +const MAINTAINERS_PATHS = [ + 'MAINTAINERS.md', + 'MAINTAINERS', + 'MAINTAINERS.txt', + 'docs/MAINTAINERS.md', +] + +async function fetchMAINTAINERS(owner: string, repo: string): Promise { + for (const p of MAINTAINERS_PATHS) { + const data = await safeGet<{ content?: string }>( + github, + `/repos/${owner}/${repo}/contents/${p}`, + ) + if (data?.content) { + return Buffer.from(data.content, 'base64').toString('utf-8') + } + } + return null +} + +interface ParsedMaintainer { + login: string + displayName: string + email: string +} + +// Extract GitHub handles and emails from free-form MAINTAINERS file +function parseMAINTAINERS(content: string): ParsedMaintainer[] { + const results: ParsedMaintainer[] = [] + const seenLogins = new Set() + + for (const line of content.split('\n')) { + const trimmed = line.trim() + if (!trimmed || trimmed.startsWith('#') || trimmed.startsWith('|--')) continue + + // Try to extract @login + const loginMatch = trimmed.match(/@([a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?)/) + const login = loginMatch ? loginMatch[1].toLowerCase() : '' + + // Try to extract email + const emailMatch = trimmed.match(/[\w.+-]+@[\w-]+\.[\w.]+/) + const email = emailMatch ? emailMatch[0] : '' + + // Try to extract display name — text before the @handle or email + const nameMatch = trimmed.match(/^[*\-\s]*([A-Z][^@<|(]+?)(?:\s*[@<|(]|$)/) + const displayName = nameMatch ? nameMatch[1].trim() : '' + + if (!login && !email) continue + if (login && seenLogins.has(login)) continue + if (login) seenLogins.add(login) + + results.push({ login, displayName, email }) + } + + return results +} + +// ─── GitHub: contributors fallback ─────────────────────────────────────────── + +interface GithubContributor { + login: string + html_url: string + contributions: number + type: string +} + +async function fetchTopContributors( + owner: string, + repo: string, + limit: number, +): Promise { + const results: GithubContributor[] = [] + let page = 1 + let hasMore = true + + while (hasMore) { + const data = await safeGet( + github, + `/repos/${owner}/${repo}/contributors?per_page=100&page=${page}&anon=false`, + ) + if (!data || data.length === 0) break + + const users = data.filter((c) => c.type === 'User') + results.push(...users) + + hasMore = data.length === 100 && (limit === 0 || results.length < limit) + if (hasMore) { + page++ + await sleep(GITHUB_DELAY_MS) + } + } + + return limit > 0 ? results.slice(0, limit) : results +} + +// ─── Concurrency pool ───────────────────────────────────────────────────────── + +async function withConcurrency( + items: T[], + limit: number, + fn: (item: T, index: number) => Promise, +): Promise { + let index = 0 + async function worker(): Promise { + while (index < items.length) { + const i = index++ + await fn(items[i], i) + } + } + await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker)) +} + +// ─── Per-package enrichment ─────────────────────────────────────────────────── + +async function enrichPackage(row: Record): Promise { + const { purl, namespace: groupId, name: artifactId } = row + + const base: Omit< + MaintainerRow, + | 'maintainer_github_login' + | 'maintainer_display_name' + | 'maintainer_email' + | 'maintainer_url' + | 'role' + | 'source' + | 'contributions' + | 'notes' + | 'github_repo' + | 'repo_source' + > = { + package_purl: purl, + package_namespace: groupId, + package_name: artifactId, + } + + const emptyMaintainer = (): MaintainerRow => ({ + ...base, + github_repo: '', + repo_source: '', + maintainer_github_login: '', + maintainer_display_name: '', + maintainer_email: '', + maintainer_url: '', + role: '', + source: '', + contributions: '', + notes: 'no maintainer found', + }) + + // 1a. Check hardcoded overrides first (fixes known wrong Libraries.io mappings) + const overrideUrl = getRepoOverride(groupId) + + // 1b. Resolve GitHub repo URL via Libraries.io (skip if junk) + const libIoUrl = await getLibIoRepoUrl(groupId, artifactId) + await sleep(LIBRARIES_IO_DELAY_MS) + const validLibIoUrl = libIoUrl && !isJunkRepoUrl(libIoUrl) ? libIoUrl : null + + // 1c. Fallback: extract SCM URL from Maven Central POM + let repoUrl = overrideUrl ?? validLibIoUrl + let repoSource = overrideUrl ? 'override' : validLibIoUrl ? 'libraries.io' : '' + if (!repoUrl) { + repoUrl = await getPomScmUrl(groupId, artifactId) + repoSource = 'pom_scm' + } + + // 1d. Last-resort: Apache namespace heuristic (org.apache.* → github.com/apache/*) + if (!repoUrl || !parseGithubRepo(repoUrl)) { + const guess = guessApacheGithubUrl(groupId, artifactId) + if (guess) { + repoUrl = guess + repoSource = 'apache_heuristic' + } + } + + if (!repoUrl) { + process.stdout.write(`(no repo url) `) + return [{ ...emptyMaintainer(), notes: 'no github repo found' }] + } + + const parsed = parseGithubRepo(repoUrl) + if (!parsed) { + process.stdout.write(`(non-github: ${repoUrl}) `) + return [{ ...emptyMaintainer(), notes: `non-github repo [${repoSource}]: ${repoUrl}` }] + } + + const { owner, repo } = parsed + const github_repo = `${owner}/${repo}` + + // 2. Try CODEOWNERS + const codeownersContent = await fetchCODEOWNERS(owner, repo) + await sleep(GITHUB_DELAY_MS) + + if (codeownersContent) { + const logins = parseCODEOWNERS(codeownersContent) + if (logins.length > 0) { + return logins.map((login) => ({ + ...base, + github_repo, + repo_source: repoSource, + maintainer_github_login: login, + maintainer_display_name: '', + maintainer_email: '', + maintainer_url: `https://github.com/${login}`, + role: 'maintainer', + source: 'codeowners', + contributions: '', + notes: '', + })) + } + } + + // 3. Try MAINTAINERS file + const maintainersContent = await fetchMAINTAINERS(owner, repo) + await sleep(GITHUB_DELAY_MS) + + if (maintainersContent) { + const maintainers = parseMAINTAINERS(maintainersContent) + if (maintainers.length > 0) { + return maintainers.map((m) => ({ + ...base, + github_repo, + repo_source: repoSource, + maintainer_github_login: m.login, + maintainer_display_name: m.displayName, + maintainer_email: m.email, + maintainer_url: m.login ? `https://github.com/${m.login}` : '', + role: 'maintainer', + source: 'maintainers_file', + contributions: '', + notes: '', + })) + } + } + + // 4. Fallback: top contributors + const contributors = await fetchTopContributors(owner, repo, TOP_CONTRIBUTORS_LIMIT) + await sleep(GITHUB_DELAY_MS) + + if (contributors.length > 0) { + return contributors.map((c) => ({ + ...base, + github_repo, + repo_source: repoSource, + maintainer_github_login: c.login, + maintainer_display_name: '', + maintainer_email: '', + maintainer_url: c.html_url, + role: 'contributor', + source: 'github_contributors', + contributions: String(c.contributions), + notes: + TOP_CONTRIBUTORS_LIMIT > 0 + ? `top ${TOP_CONTRIBUTORS_LIMIT} contributors` + : 'all contributors', + })) + } + + return [ + { + ...emptyMaintainer(), + github_repo, + repo_source: repoSource, + notes: 'repo found but no maintainer data', + }, + ] +} + +// ─── Main ───────────────────────────────────────────────────────────────────── + +async function main() { + if (!LIBRARIES_IO_KEY) { + console.error('Missing LIBRARIES_IO_API_KEY env var') + process.exit(1) + } + if (!GITHUB_TOKEN) { + console.warn( + 'Warning: GITHUB_TOKEN not set — GitHub API rate limit will be very low (60 req/hr)', + ) + } + + const inputPath = process.argv[2] + if (!inputPath) { + console.error( + 'Usage: LIBRARIES_IO_API_KEY=xxx GITHUB_TOKEN=yyy tsx src/maven/scripts/fetchMavenMaintainers.ts [output.csv]', + ) + process.exit(1) + } + + const outputPath = + process.argv[3] ?? + path.join(path.dirname(inputPath), path.basename(inputPath, '.csv') + '_maintainers.csv') + + const content = fs.readFileSync(inputPath, 'utf-8') + const rows = parseCsv(content).filter((r) => r.ecosystem === 'maven' && r.purl) + + console.log(`Found ${rows.length} maven packages — output: ${outputPath}`) + if (!GITHUB_TOKEN) console.warn(' (no GitHub token — using unauthenticated API, very slow)') + + const CSV_HEADER = + 'package_purl,package_namespace,package_name,github_repo,repo_source,maintainer_github_login,maintainer_display_name,maintainer_email,maintainer_url,role,source,contributions,notes' + + const resultsByIndex: MaintainerRow[][] = new Array(rows.length) + const stats = { codeowners: 0, maintainers_file: 0, contributors: 0, none: 0 } + let completed = 0 + + // Concurrency 3 — Libraries.io rate limit is the bottleneck + await withConcurrency(rows, 3, async (row, i) => { + const pkg = `${row.namespace}/${row.name}` + process.stdout.write(`[${i + 1}/${rows.length}] ${pkg} ... `) + + const results = await enrichPackage(row) + resultsByIndex[i] = results + + const source = results[0]?.source ?? '' + if (source === 'codeowners') stats.codeowners++ + else if (source === 'maintainers_file') stats.maintainers_file++ + else if (source === 'github_contributors') stats.contributors++ + else stats.none++ + + console.log(`${results.length} maintainer(s) [${source || 'none'}]`) + + // Checkpoint every 50 packages — write rows in original input order + completed++ + if (completed % 50 === 0) { + const checkpointLines = [CSV_HEADER, ...resultsByIndex.flat().map(toCsvLine)] + fs.writeFileSync(outputPath, checkpointLines.join('\n')) + console.log(` ✓ checkpoint saved (${completed} done)`) + } + }) + + const lines = [CSV_HEADER, ...resultsByIndex.flat().map(toCsvLine)] + fs.writeFileSync(outputPath, lines.join('\n')) + console.log(`\nDone. ${lines.length - 1} rows written to: ${outputPath}`) + + console.log( + ` CODEOWNERS: ${stats.codeowners} | MAINTAINERS file: ${stats.maintainers_file} | contributors fallback: ${stats.contributors} | not found: ${stats.none}`, + ) +} + +main().catch((err) => { + console.error(err) + process.exit(1) +}) diff --git a/services/apps/packages_worker/src/maven/scripts/importMaintainersFromCsv.ts b/services/apps/packages_worker/src/maven/scripts/importMaintainersFromCsv.ts new file mode 100644 index 0000000000..cae07d38bf --- /dev/null +++ b/services/apps/packages_worker/src/maven/scripts/importMaintainersFromCsv.ts @@ -0,0 +1,196 @@ +/** + * Imports maintainer data from the CSV produced by fetchMavenMaintainers.ts + * into the osspckgs database (maintainers + package_maintainers tables). + * + * Safe to run multiple times — all writes are INSERT ... ON CONFLICT DO NOTHING. + * Existing maintainer data from POM extraction is never deleted. + * + * Generates a rollback.sql file next to the input CSV so the import can be + * fully reversed: psql $OSSPCKGS_DB_URL -f rollback.sql + * + * Flags: + * --dry-run Print what would be inserted without touching the DB + * + * Usage: + * tsx src/maven/scripts/importMaintainersFromCsv.ts [--dry-run] + */ +import * as fs from 'fs' +import * as path from 'path' + +import { + insertPackageMaintainerLink, + upsertMaintainer, +} from '@crowd/data-access-layer/src/osspckgs/maintainers' +import { findPackageIdsByPurl } from '@crowd/data-access-layer/src/osspckgs/packages' +import { pgpQx } from '@crowd/data-access-layer/src/queryExecutor' + +import { getPackagesDbConnection } from '../../db' + +import { parseCsv } from './csv' + +// ─── Role normalisation ─────────────────────────────────────────────────────── + +function normaliseRole(raw: string): 'author' | 'maintainer' | 'contributor' | null { + if (raw === 'author') return 'author' + if (raw === 'maintainer') return 'maintainer' + if (raw === 'contributor') return 'contributor' + return null +} + +// ─── Main ───────────────────────────────────────────────────────────────────── + +async function main() { + const args = process.argv.slice(2) + const dryRun = args.includes('--dry-run') + const inputPath = args.find((a) => !a.startsWith('--')) + + if (!inputPath) { + console.error( + 'Usage: tsx src/maven/scripts/importMaintainersFromCsv.ts [--dry-run]', + ) + process.exit(1) + } + + const rollbackPath = path.join( + path.dirname(inputPath), + path.basename(inputPath, '.csv') + '_rollback.sql', + ) + + console.log(`Input: ${inputPath}`) + console.log(`Rollback: ${rollbackPath}`) + console.log(`Mode: ${dryRun ? 'DRY RUN (no DB writes)' : 'LIVE'}`) + console.log() + + const content = fs.readFileSync(inputPath, 'utf-8') + const rows = parseCsv(content).filter((r) => r.package_purl && r.maintainer_github_login) + + console.log(`Rows with maintainer data: ${rows.length}`) + + // Group by purl + const byPurl = new Map() + for (const r of rows) { + const list = byPurl.get(r.package_purl) ?? [] + list.push(r) + byPurl.set(r.package_purl, list) + } + console.log(`Unique packages to process: ${byPurl.size}`) + + if (dryRun) { + // Just print a sample and exit + let i = 0 + const entries = Array.from(byPurl.entries()) + for (const [purl, maintainers] of entries) { + console.log(` ${purl} → ${maintainers.length} maintainers`) + if (++i >= 10) { + console.log(` ... (${byPurl.size - 10} more packages)`) + break + } + } + console.log('\nDry run complete — no changes made.') + return + } + + const conn = await getPackagesDbConnection() + const qx = pgpQx(conn) + + // Batch-resolve all purls to package IDs in one query + const purlList = [...byPurl.keys()] + const purlToId = await findPackageIdsByPurl(qx, purlList) + + const missing = purlList.filter((p) => !purlToId.has(p)) + if (missing.length > 0) { + console.warn(`\nWarning: ${missing.length} packages not found in DB (will be skipped):`) + missing.slice(0, 10).forEach((p) => console.warn(` ${p}`)) + if (missing.length > 10) console.warn(` ... and ${missing.length - 10} more`) + } + + const insertedPmIds: number[] = [] + let upsertedMaintainers = 0 + let insertedLinks = 0 + let skippedLinks = 0 + let skippedPackages = 0 + + let processed = 0 + const total = byPurl.size + + for (const [purl, maintainerRows] of Array.from(byPurl.entries())) { + const packageId = purlToId.get(purl) + if (!packageId) { + skippedPackages++ + continue + } + + await conn.tx(async (t) => { + const tqx = pgpQx(t) + + for (const r of maintainerRows) { + // 1. Upsert the maintainer profile (github_login included via DAL) + const { id: maintainerId } = await upsertMaintainer(tqx, { + ecosystem: 'maven', + username: r.maintainer_github_login, + displayName: r.maintainer_display_name || null, + url: r.maintainer_url || null, + email: r.maintainer_email || null, + githubLogin: r.maintainer_github_login || null, + }) + upsertedMaintainers++ + + // 2. Insert the package→maintainer link (non-destructive, via DAL) + const role = normaliseRole(r.role) + const insertedId = await insertPackageMaintainerLink( + tqx, + packageId, + maintainerId, + role, + 'manual_csv', + ) + + if (insertedId !== null) { + insertedLinks++ + insertedPmIds.push(insertedId) + } else { + skippedLinks++ + } + } + }) + + processed++ + if (processed % 50 === 0 || processed === total) { + console.log( + `[${processed}/${total}] maintainers=${upsertedMaintainers} links_inserted=${insertedLinks} links_skipped=${skippedLinks}`, + ) + } + } + + await (conn as unknown as { $pool: { end: () => Promise } }).$pool.end() + + // Write rollback file + const rollbackSql = [ + `-- Rollback for import: ${path.basename(inputPath)}`, + `-- Generated: ${new Date().toISOString()}`, + `-- Deletes only the package_maintainers rows inserted by this import.`, + `-- Maintainer profiles in the maintainers table are left intact.`, + ``, + insertedPmIds.length > 0 + ? `DELETE FROM package_maintainers WHERE id IN (${insertedPmIds.join(', ')});` + : `-- Nothing was inserted — no rows to roll back.`, + ``, + `-- Rows deleted: ${insertedPmIds.length}`, + ].join('\n') + + fs.writeFileSync(rollbackPath, rollbackSql) + + console.log(`\n✓ Done.`) + console.log(` Packages processed: ${processed}`) + console.log(` Packages skipped (not in DB): ${skippedPackages}`) + console.log(` Maintainer upserts: ${upsertedMaintainers}`) + console.log(` package_maintainers inserted: ${insertedLinks}`) + console.log(` package_maintainers skipped (already existed): ${skippedLinks}`) + console.log(`\n Rollback file: ${rollbackPath}`) + console.log(` To revert: psql $OSSPCKGS_DB_URL -f ${rollbackPath}`) +} + +main().catch((err) => { + console.error(err) + process.exit(1) +}) diff --git a/services/libs/data-access-layer/src/osspckgs/maintainers.ts b/services/libs/data-access-layer/src/osspckgs/maintainers.ts index 8d2c320b55..e15a754fd5 100644 --- a/services/libs/data-access-layer/src/osspckgs/maintainers.ts +++ b/services/libs/data-access-layer/src/osspckgs/maintainers.ts @@ -13,32 +13,56 @@ export async function upsertMaintainer( const row = await qx.selectOne( ` WITH old AS ( - SELECT display_name, url, email + SELECT display_name, url, email, github_login FROM maintainers WHERE ecosystem = $(ecosystem) AND username = $(username) ), ins AS ( - INSERT INTO maintainers (ecosystem, username, display_name, url, email, created_at, updated_at) - VALUES ($(ecosystem), $(username), $(displayName), $(url), $(email), NOW(), NOW()) + INSERT INTO maintainers (ecosystem, username, display_name, url, email, github_login, created_at, updated_at) + VALUES ($(ecosystem), $(username), $(displayName), $(url), $(email), $(githubLogin), NOW(), NOW()) ON CONFLICT (ecosystem, username) DO UPDATE SET display_name = COALESCE(EXCLUDED.display_name, maintainers.display_name), url = COALESCE(EXCLUDED.url, maintainers.url), email = COALESCE(EXCLUDED.email, maintainers.email), + github_login = COALESCE(EXCLUDED.github_login, maintainers.github_login), updated_at = NOW() - RETURNING id, display_name, url, email + RETURNING id, display_name, url, email, github_login ) SELECT ins.id, array_remove(ARRAY[ CASE WHEN o.display_name IS DISTINCT FROM ins.display_name THEN 'maintainers.display_name' END, CASE WHEN o.url IS DISTINCT FROM ins.url THEN 'maintainers.url' END, - CASE WHEN o.email IS DISTINCT FROM ins.email THEN 'maintainers.email' END + CASE WHEN o.email IS DISTINCT FROM ins.email THEN 'maintainers.email' END, + CASE WHEN o.github_login IS DISTINCT FROM ins.github_login THEN 'maintainers.github_login' END ], NULL) AS changed_fields FROM ins LEFT JOIN old o ON true `, - item, + { ...item, githubLogin: item.githubLogin ?? null }, ) return { id: row.id as number, changedFields: row.changed_fields as string[] } } +/** + * Inserts a package→maintainer link if it doesn't already exist. + * Returns the id of the newly inserted row, or null if it already existed. + * Non-destructive: never deletes or modifies existing links. + */ +export async function insertPackageMaintainerLink( + qx: QueryExecutor, + packageId: number, + maintainerId: number, + role: 'author' | 'maintainer' | 'contributor' | null, + ingestionSource?: string | null, +): Promise { + const row = await qx.selectOneOrNone( + `INSERT INTO package_maintainers (package_id, maintainer_id, role, ingestion_source, created_at, updated_at) + VALUES ($(packageId), $(maintainerId), $(role), $(ingestionSource), NOW(), NOW()) + ON CONFLICT (package_id, maintainer_id) DO NOTHING + RETURNING id`, + { packageId, maintainerId, role, ingestionSource: ingestionSource ?? null }, + ) + return row?.id ?? null +} + /** * Replaces all maintainer links for a package with the given list. * Deletes links that are no longer present and inserts/updates new ones. @@ -50,19 +74,33 @@ export async function replacePackageMaintainers( links: Array>, ): Promise { const before: Array<{ maintainer_id: number; role: string | null }> = await qx.select( - `SELECT maintainer_id, role FROM package_maintainers WHERE package_id = $(packageId)`, + `SELECT maintainer_id, role FROM package_maintainers + WHERE package_id = $(packageId) AND ingestion_source IS DISTINCT FROM 'manual_csv'`, { packageId }, ) const beforeMap = new Map(before.map((r) => [r.maintainer_id, r.role])) - await qx.result(`DELETE FROM package_maintainers WHERE package_id = $(packageId)`, { packageId }) + await qx.result( + `DELETE FROM package_maintainers + WHERE package_id = $(packageId) AND ingestion_source IS DISTINCT FROM 'manual_csv'`, + { packageId }, + ) + + // Surviving manual_csv links must be excluded from the changedFields diff — + // they are pre-existing and should not appear as "added" in afterMap comparisons. + const manualCsvRows: Array<{ maintainer_id: number }> = await qx.select( + `SELECT maintainer_id FROM package_maintainers + WHERE package_id = $(packageId) AND ingestion_source = 'manual_csv'`, + { packageId }, + ) + const manualCsvIds = new Set(manualCsvRows.map((r) => r.maintainer_id)) const afterMap = new Map() for (const { maintainerId, role } of links) { await qx.result( `INSERT INTO package_maintainers (package_id, maintainer_id, role, created_at, updated_at) VALUES ($(packageId), $(maintainerId), $(role), NOW(), NOW()) - ON CONFLICT (package_id, maintainer_id) DO UPDATE SET role = EXCLUDED.role, updated_at = NOW()`, + ON CONFLICT (package_id, maintainer_id) DO NOTHING`, { packageId, maintainerId, role }, ) afterMap.set(maintainerId, role) @@ -73,6 +111,7 @@ export async function replacePackageMaintainers( if (!afterMap.has(id)) changed.add('package_maintainers.maintainer_id') } for (const [id, role] of afterMap) { + if (manualCsvIds.has(id)) continue if (!beforeMap.has(id)) changed.add('package_maintainers.maintainer_id') else if (beforeMap.get(id) !== role) changed.add('package_maintainers.role') } diff --git a/services/libs/data-access-layer/src/osspckgs/types.ts b/services/libs/data-access-layer/src/osspckgs/types.ts index bcefe84fbe..c18767e4ef 100644 --- a/services/libs/data-access-layer/src/osspckgs/types.ts +++ b/services/libs/data-access-layer/src/osspckgs/types.ts @@ -43,6 +43,7 @@ export type IDbMaintainerUpsert = { displayName: string | null url: string | null email: string | null + githubLogin?: string | null } // ─── package_maintainers ────────────────────────────────────────────────────── @@ -50,7 +51,8 @@ export type IDbMaintainerUpsert = { export type IDbPackageMaintainerUpsert = { packageId: number maintainerId: number - role: 'author' | 'maintainer' | null + role: 'author' | 'maintainer' | 'contributor' | null + ingestionSource?: string | null } // ─── versions ───────────────────────────────────────────────────────────────── diff --git a/services/libs/tinybird/datasources/packageMaintainers.datasource b/services/libs/tinybird/datasources/packageMaintainers.datasource index c51ed3ee37..1170e98344 100644 --- a/services/libs/tinybird/datasources/packageMaintainers.datasource +++ b/services/libs/tinybird/datasources/packageMaintainers.datasource @@ -5,7 +5,8 @@ DESCRIPTION > - `id` is the internal primary key. - `packageId` links to the parent packages row. - `maintainerId` links to the parent maintainers row. - - `role` is the maintainer's role on the package: 'author', 'maintainer', or empty string if not specified. + - `role` is the maintainer's role on the package: 'author', 'maintainer', 'contributor', or empty string if not specified. + - `ingestionSource` identifies where this link came from (e.g. 'manual_csv'); empty string for auto-enriched rows. - `createdAt` and `updatedAt` are row-level audit timestamps for Tinybird watermark-based sync. SCHEMA > @@ -13,6 +14,7 @@ SCHEMA > `packageId` UInt64 `json:$.record.package_id`, `maintainerId` UInt64 `json:$.record.maintainer_id`, `role` String `json:$.record.role` DEFAULT '', + `ingestionSource` String `json:$.record.ingestion_source` DEFAULT '', `createdAt` DateTime64(3) `json:$.record.created_at`, `updatedAt` DateTime64(3) `json:$.record.updated_at`