feat: wire security contacts pipeline + DB write

Signed-off-by: Mouad BANI <mouad-mb@outlook.com>

Author: mbani01

services/apps/packages_worker/src/config.ts

@@ -50,6 +50,8 @@ export function getSecurityContactsConfig() {
   return {
     // A repo is re-evaluated once its contacts are older than this many hours.
     updateIntervalHours: requireEnvInt('SECURITY_CONTACTS_UPDATE_INTERVAL_HOURS'),
+    // Sent on all registry calls; crates.io rejects requests without an identifying UA.
+    userAgent: requireEnv('SECURITY_CONTACTS_USER_AGENT'),
     // Lower than the enricher: each repo fans out to several HTTP calls across extractors
     concurrency: parseInt(process.env.SECURITY_CONTACTS_CONCURRENCY ?? '20', 10),
     fetchTimeoutMs: parseInt(process.env.SECURITY_CONTACTS_FETCH_TIMEOUT_MS ?? '10000', 10),

services/apps/packages_worker/src/security-contacts/extractors/http.ts

@@ -1,6 +1,12 @@
 export const RAW_BASE = 'https://raw.githubusercontent.com'
 export const GITHUB_API = 'https://api.github.com'
 
+// crates.io rejects requests without a descriptive User-Agent (HTTP 403); harmless elsewhere.
+// https://crates.io/policies#crawlers
+export function registryHeaders(userAgent: string): Record<string, string> {
+  return { 'User-Agent': userAgent }
+}
+
 export interface FetchTextResult {
   status: number
   text: string | null
@@ -48,11 +54,17 @@ export function githubAuthHeaders(token: string): Record<string, string> {
 }
 
 const EMAIL_RE = /^[^@\s]+@[^@\s]+\.[^@\s]+$/
+const EMAIL_GLOBAL_RE = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g
 
 export function isEmail(value: string): boolean {
   return EMAIL_RE.test(value.trim())
 }
 
+// Pulls all email addresses out of free-form strings like "Name <a@b.com>, Name2 <c@d.com>".
+export function extractEmails(value: string): string[] {
+  return value.match(EMAIL_GLOBAL_RE) ?? []
+}
+
 /** Returns the login if the URL is a bare github profile (github.com/<login>), else null. */
 export function githubHandleFromUrl(value: string): string | null {
   const m = value.trim().match(/^https?:\/\/github\.com\/([^/\s]+)\/?$/i)

services/apps/packages_worker/src/security-contacts/extractors/pvr.ts

@@ -43,7 +43,8 @@ export const extractPvr: Extractor = async (target, deps) => {
   }
 
   // The endpoint works unauthenticated, but we use the token pool for rate-limit budget.
-  const headers = deps.getToken ? githubAuthHeaders(await deps.getToken()) : {}
+  const token = deps.getToken ? await deps.getToken() : null
+  const headers = token ? githubAuthHeaders(token) : {}
   const { json } = await fetchJson(
     `${GITHUB_API}/repos/${owner}/${name}/private-vulnerability-reporting`,
     deps.fetchTimeoutMs,

services/apps/packages_worker/src/security-contacts/extractors/registry/cargo.ts

@@ -0,0 +1,60 @@
+import { ExtractorResult, ProvenanceEntry, RawContact } from '../../types'
+import { fetchJson, registryHeaders } from '../http'
+
+import { ParsedPurl } from './purl'
+
+const SOURCE = 'crates.io'
+
+// crates.io policy: max 1 request/second. Serialize calls process-wide.
+// https://crates.io/policies#crawlers
+let nextAllowedAt = 0
+async function throttle(): Promise<void> {
+  const now = Date.now()
+  const waitMs = Math.max(0, nextAllowedAt - now)
+  nextAllowedAt = Math.max(now, nextAllowedAt) + 1000
+  if (waitMs > 0) await new Promise((r) => setTimeout(r, waitMs))
+}
+
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+export function mapCargoOwners(doc: unknown, sourceUrl: string, fetchedAt: string): RawContact[] {
+  if (!doc || typeof doc !== 'object') return []
+  const users = (doc as any).users
+  if (!Array.isArray(users)) return []
+
+  const prov = (): ProvenanceEntry[] => [
+    { source: SOURCE, sourceTier: 'B', path: sourceUrl, fetchedAt },
+  ]
+  const contacts: RawContact[] = []
+  const seen = new Set<string>()
+  for (const u of users) {
+    const login = u?.login
+    // crates.io logins like "github:org:team" are teams, not personal handles.
+    if (typeof login !== 'string' || login.includes(':')) continue
+    const key = login.toLowerCase()
+    if (seen.has(key)) continue
+    seen.add(key)
+    contacts.push({
+      channel: 'github-handle',
+      value: login,
+      name: typeof u.name === 'string' ? u.name : undefined,
+      role: 'maintainer',
+      tier: 'B',
+      provenance: prov(),
+    })
+  }
+  return contacts
+}
+
+export async function fetchCargo(
+  parsed: ParsedPurl,
+  timeoutMs: number,
+  userAgent: string,
+): Promise<ExtractorResult> {
+  // crates.io does not expose author emails; owners give GitHub handles.
+  await throttle()
+  const url = `https://crates.io/api/v1/crates/${encodeURIComponent(parsed.name)}/owners`
+  const { json } = await fetchJson(url, timeoutMs, registryHeaders(userAgent))
+  if (!json) return { contacts: [], policies: {} }
+  return { contacts: mapCargoOwners(json, url, new Date().toISOString()), policies: {} }
+}

services/apps/packages_worker/src/security-contacts/extractors/registry/composer.ts

@@ -0,0 +1,72 @@
+import { ExtractorResult, ProvenanceEntry, RawContact, RepoPolicies } from '../../types'
+import { fetchJson, isEmail, registryHeaders } from '../http'
+
+import { ParsedPurl } from './purl'
+
+const SOURCE = 'packagist'
+
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+export function mapComposer(
+  doc: unknown,
+  fullName: string,
+  sourceUrl: string,
+  fetchedAt: string,
+): ExtractorResult {
+  const contacts: RawContact[] = []
+  const policies: Partial<RepoPolicies> = {}
+  if (!doc || typeof doc !== 'object') return { contacts, policies }
+
+  const versions = (doc as any).packages?.[fullName]
+  const latest = Array.isArray(versions) ? versions[0] : undefined
+  if (!latest || typeof latest !== 'object') return { contacts, policies }
+
+  const prov = (): ProvenanceEntry[] => [
+    { source: SOURCE, sourceTier: 'B', path: sourceUrl, fetchedAt },
+  ]
+  const seen = new Set<string>()
+  const add = (
+    channel: RawContact['channel'],
+    value: string,
+    role: RawContact['role'],
+    name?: string,
+  ): void => {
+    const key = `${channel}:${value.toLowerCase()}`
+    if (seen.has(key)) return
+    seen.add(key)
+    contacts.push({ channel, value, name, role, tier: 'B', provenance: prov() })
+  }
+
+  for (const a of Array.isArray(latest.authors) ? latest.authors : []) {
+    if (typeof a?.email === 'string' && isEmail(a.email)) {
+      add('email', a.email, 'maintainer', typeof a.name === 'string' ? a.name : undefined)
+    }
+  }
+
+  const support = latest.support
+  if (support && typeof support === 'object') {
+    // support.security is Composer's dedicated security channel.
+    if (typeof support.security === 'string' && /^https?:\/\//i.test(support.security)) {
+      policies.securityPolicyUrl = support.security
+      add('url', support.security, 'security-team')
+    }
+    if (typeof support.email === 'string' && isEmail(support.email)) {
+      add('email', support.email, 'security-team')
+    }
+  }
+
+  return { contacts, policies }
+}
+
+export async function fetchComposer(
+  parsed: ParsedPurl,
+  timeoutMs: number,
+  userAgent: string,
+): Promise<ExtractorResult> {
+  if (!parsed.namespace) return { contacts: [], policies: {} }
+  const fullName = `${parsed.namespace}/${parsed.name}`
+  const url = `https://repo.packagist.org/p2/${fullName}.json`
+  const { json } = await fetchJson(url, timeoutMs, registryHeaders(userAgent))
+  if (!json) return { contacts: [], policies: {} }
+  return mapComposer(json, fullName, url, new Date().toISOString())
+}

services/apps/packages_worker/src/security-contacts/extractors/registry/index.ts

@@ -1,17 +1,29 @@
-import { Extractor, RawContact, RepoPackage, RepoPolicies } from '../../types'
+import { Extractor, ExtractorResult, RawContact, RepoPolicies } from '../../types'
 
+import { fetchCargo } from './cargo'
+import { fetchComposer } from './composer'
+import { fetchMaven } from './maven'
 import { fetchNpm } from './npm'
-import { parsePurl } from './purl'
+import { fetchNuget } from './nuget'
+import { ParsedPurl, parsePurl } from './purl'
+import { fetchPypi } from './pypi'
+import { fetchRubygems } from './rubygems'
 
 type EcosystemFetcher = (
-  parsed: ReturnType<typeof parsePurl> & object,
-  pkg: RepoPackage,
+  parsed: ParsedPurl,
   timeoutMs: number,
-) => Promise<RawContact[]>
+  userAgent: string,
+) => Promise<ExtractorResult>
 
-// Keyed by the lowercased packages.ecosystem value. go has no manifest contacts.
+// Keyed by the lowercased packages.ecosystem value. go has no package-manifest contacts.
 const FETCHERS: Record<string, EcosystemFetcher> = {
   npm: fetchNpm,
+  pypi: fetchPypi,
+  maven: fetchMaven,
+  cargo: fetchCargo,
+  nuget: fetchNuget,
+  rubygems: fetchRubygems,
+  composer: fetchComposer,
 }
 
 export const extractManifest: Extractor = async (target, deps) => {
@@ -30,7 +42,13 @@ export const extractManifest: Extractor = async (target, deps) => {
     if (!parsed) continue
 
     try {
-      contacts.push(...(await fetcher(parsed, pkg, deps.fetchTimeoutMs)))
+      const result = await fetcher(parsed, deps.fetchTimeoutMs, deps.userAgent)
+      contacts.push(...result.contacts)
+      for (const [key, value] of Object.entries(result.policies)) {
+        if (!(policies as Record<string, unknown>)[key] && value != null) {
+          ;(policies as Record<string, unknown>)[key] = value
+        }
+      }
     } catch {
       // one bad package must not sink the rest
       continue

services/apps/packages_worker/src/security-contacts/extractors/registry/maven.ts

@@ -0,0 +1,90 @@
+import { XMLParser } from 'fast-xml-parser'
+
+import { ExtractorResult, ProvenanceEntry, RawContact } from '../../types'
+import { fetchText, isEmail, registryHeaders } from '../http'
+
+import { ParsedPurl } from './purl'
+
+const SOURCE = 'maven-pom'
+const BASE = 'https://repo1.maven.org/maven2'
+const parser = new XMLParser({ ignoreAttributes: true })
+
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+function asArray<T>(x: T | T[] | undefined): T[] {
+  if (x === undefined || x === null) return []
+  return Array.isArray(x) ? x : [x]
+}
+
+export function mapMavenPom(xml: string, sourceUrl: string, fetchedAt: string): RawContact[] {
+  let doc: any
+  try {
+    doc = parser.parse(xml)
+  } catch {
+    return []
+  }
+  const project = doc?.project
+  if (!project) return []
+
+  const prov = (): ProvenanceEntry[] => [
+    { source: SOURCE, sourceTier: 'B', path: sourceUrl, fetchedAt },
+  ]
+  const contacts: RawContact[] = []
+  const seen = new Set<string>()
+  for (const dev of asArray(project.developers?.developer)) {
+    const email = dev?.email
+    if (typeof email !== 'string' || !isEmail(email)) continue
+    const key = email.toLowerCase()
+    if (seen.has(key)) continue
+    seen.add(key)
+    contacts.push({
+      channel: 'email',
+      value: email,
+      name: typeof dev.name === 'string' ? dev.name : undefined,
+      role: 'maintainer',
+      tier: 'B',
+      provenance: prov(),
+    })
+  }
+  return contacts
+}
+
+async function resolveVersion(
+  groupPath: string,
+  artifact: string,
+  timeoutMs: number,
+  userAgent: string,
+): Promise<string | null> {
+  const url = `${BASE}/${groupPath}/${artifact}/maven-metadata.xml`
+  const { text } = await fetchText(url, timeoutMs, registryHeaders(userAgent))
+  if (!text) return null
+  let doc: any
+  try {
+    doc = parser.parse(text)
+  } catch {
+    return null
+  }
+  const versioning = doc?.metadata?.versioning
+  const release = versioning?.release ?? versioning?.latest
+  if (typeof release === 'string') return release
+  const versions = asArray(versioning?.versions?.version)
+  return versions.length ? String(versions[versions.length - 1]) : null
+}
+
+export async function fetchMaven(
+  parsed: ParsedPurl,
+  timeoutMs: number,
+  userAgent: string,
+): Promise<ExtractorResult> {
+  if (!parsed.namespace) return { contacts: [], policies: {} }
+  const groupPath = parsed.namespace.replace(/\./g, '/')
+  const artifact = parsed.name
+
+  const version = await resolveVersion(groupPath, artifact, timeoutMs, userAgent)
+  if (!version) return { contacts: [], policies: {} }
+
+  const url = `${BASE}/${groupPath}/${artifact}/${version}/${artifact}-${version}.pom`
+  const { text } = await fetchText(url, timeoutMs, registryHeaders(userAgent))
+  if (!text) return { contacts: [], policies: {} }
+  return { contacts: mapMavenPom(text, url, new Date().toISOString()), policies: {} }
+}

services/apps/packages_worker/src/security-contacts/extractors/registry/npm.ts

@@ -1,5 +1,5 @@
-import { ProvenanceEntry, RawContact, RepoPackage } from '../../types'
-import { fetchJson, isEmail } from '../http'
+import { ExtractorResult, ProvenanceEntry, RawContact } from '../../types'
+import { fetchJson, isEmail, registryHeaders } from '../http'
 
 import { ParsedPurl } from './purl'
 
@@ -50,11 +50,11 @@ export function mapNpm(doc: unknown, sourceUrl: string, fetchedAt: string): RawC
 
 export async function fetchNpm(
   parsed: ParsedPurl,
-  _pkg: RepoPackage,
   timeoutMs: number,
-): Promise<RawContact[]> {
+  userAgent: string,
+): Promise<ExtractorResult> {
   const url = `https://registry.npmjs.org/${npmPackagePath(parsed)}`
-  const { json } = await fetchJson(url, timeoutMs)
-  if (!json) return []
-  return mapNpm(json, url, new Date().toISOString())
+  const { json } = await fetchJson(url, timeoutMs, registryHeaders(userAgent))
+  if (!json) return { contacts: [], policies: {} }
+  return { contacts: mapNpm(json, url, new Date().toISOString()), policies: {} }
 }