feat: improve pom extractor performances

Signed-off-by: Umberto Sgueglia <usgueglia@contractor.linuxfoundation.org>

Author: ulemons

backend/.env.dist.local

@@ -206,3 +206,27 @@ POM_FETCHER_NON_CRITICAL_CONCURRENCY=20
 # pom-fetcher critical (HTTP)
 POM_FETCHER_BATCH_SIZE=100
 POM_FETCHER_CONCURRENCY=5
+# ── non-critical DB-only (usato quando DIRECT_POM_FOR_ALL=false) ──────────────
+# POM_FETCHER_NON_CRITICAL_BATCH_SIZE=500
+POM_FETCHER_NON_CRITICAL_CONCURRENCY=20    # solo DB writes, nessun HTTP
+POM_FETCHER_NON_CRITICAL_REFRESH_DAYS=1
+
+# ── non-critical HTTP / direct-pom (usato quando DIRECT_POM_FOR_ALL=true) ─────
+# POM_FETCHER_NON_CRITICAL_POM_CONCURRENCY=5 
+# POM_FETCHER_NON_CRITICAL_POM_REFRESH_DAYS=1
+
+# ── critical HTTP / full-pom (sempre attivo) ──────────────────────────────────
+# POM_FETCHER_BATCH_SIZE=100
+# POM_FETCHER_CONCURRENCY=5
+# POM_FETCHER_FULL_REFRESH_DAYS=1
+
+# ── modalità e rate limiting ──────────────────────────────────────────────────
+# POM_FETCHER_DIRECT_POM_FOR_ALL=true
+# POM_FETCHER_GROUP_DELAY_MS=200
+
+
+POM_FETCHER_BATCH_SIZE=50
+POM_FETCHER_CONCURRENCY=3
+POM_FETCHER_REFRESH_DAYS=1
+POM_FETCHER_GROUP_DELAY_MS=500
+POM_FETCHER_IDLE_SLEEP_SEC=3600

scripts/services/pom-fetcher.yaml

@@ -8,8 +8,9 @@ x-env-args: &env-args
   SUPPRESS_NO_CONFIG_WARNING: 'true'
   LOG_LEVEL: 'info'
   POM_FETCHER_BATCH_SIZE: '50'
-  POM_FETCHER_CONCURRENCY: '3'
-  POM_FETCHER_STALE_DAYS: '7'
+  POM_FETCHER_CONCURRENCY: '5'
+  POM_FETCHER_REFRESH_DAYS: '1'
+  POM_FETCHER_GROUP_DELAY_MS: '200'
   POM_FETCHER_IDLE_SLEEP_SEC: '3600'
 
 services:

services/apps/packages_worker/src/bin/pom-fetcher.ts

@@ -21,10 +21,7 @@ const main = async () => {
   log.info('pom-fetcher starting...')
 
   const config = getPomFetcherConfig()
-  log.info(
-    { batchSize: config.batchSize, concurrency: config.concurrency, fullRefreshDays: config.fullRefreshDays },
-    'Config loaded',
-  )
+  log.info(config, 'Config loaded')
 
   const qx = await getPackagesDb()
   await qx.selectOne('SELECT 1')

services/apps/packages_worker/src/config.ts

@@ -35,14 +35,10 @@ export function getEnricherConfig() {
 
 export function getPomFetcherConfig() {
   return {
-    // critical packages — HTTP-bound, keep low
     batchSize: parseInt(process.env.POM_FETCHER_BATCH_SIZE ?? '50', 10),
-    concurrency: parseInt(process.env.POM_FETCHER_CONCURRENCY ?? '3', 10),
-    fullRefreshDays: parseInt(process.env.POM_FETCHER_FULL_REFRESH_DAYS ?? '90', 10),
-    // non-critical packages — DB-only, can go much higher
-    nonCriticalBatchSize: parseInt(process.env.POM_FETCHER_NON_CRITICAL_BATCH_SIZE ?? '500', 10),
-    nonCriticalConcurrency: parseInt(process.env.POM_FETCHER_NON_CRITICAL_CONCURRENCY ?? '20', 10),
-    nonCriticalRefreshDays: parseInt(process.env.POM_FETCHER_NON_CRITICAL_REFRESH_DAYS ?? '180', 10),
+    concurrency: parseInt(process.env.POM_FETCHER_CONCURRENCY ?? '5', 10),
+    refreshDays: parseInt(process.env.POM_FETCHER_REFRESH_DAYS ?? '1', 10),
+    groupDelayMs: parseInt(process.env.POM_FETCHER_GROUP_DELAY_MS ?? '200', 10),
     idleSleepSec: parseInt(process.env.POM_FETCHER_IDLE_SLEEP_SEC ?? '3600', 10),
   }
 }

services/apps/packages_worker/src/pom-fetcher/README.md

@@ -11,7 +11,6 @@ import { XMLParser } from 'fast-xml-parser'
 export interface PomMaintainer {
   username: string | null
   displayName: string | null
-  /** Raw email from POM — hash with SHA-256 before storing (GDPR) */
   email: string | null
   url: string | null
   role: 'author' | 'maintainer'
@@ -55,7 +54,7 @@ interface PomPerson {
 // ─── Config ───────────────────────────────────────────────────────────────────
 
 const MAVEN_REPO = 'https://repo1.maven.org/maven2'
-const MAX_PARENT_HOPS = 5
+export const MAX_PARENT_HOPS = 7
 const REQUEST_TIMEOUT_MS = 15_000
 
 const parser = new XMLParser({
@@ -80,8 +79,10 @@ async function getWithRetry(url: string): Promise<string> {
       const res = await axios.get<string>(url, { responseType: 'text', timeout: REQUEST_TIMEOUT_MS })
       return res.data
     } catch (err) {
-      if (axios.isAxiosError(err) && err.response?.status === 429) {
-        if (attempt < MAX_RETRIES) {
+      if (axios.isAxiosError(err)) {
+        const status = err.response?.status
+        // 429 = explicit rate limit, 403 = CDN throttle (Maven Central uses both)
+        if ((status === 429 || status === 403) && attempt < MAX_RETRIES) {
           const delay = RETRY_BASE_MS * 2 ** attempt + Math.random() * 500
           await sleep(delay)
           continue
@@ -195,10 +196,65 @@ async function resolveWithInheritance(
   }
 }
 
-// ─── Public entry point ───────────────────────────────────────────────────────
+// ─── Public entry points ──────────────────────────────────────────────────────
 
 /**
- * Fetches and resolves POM metadata for the given Maven artifact.
+ * Fetches only the root POM without following the parent chain.
+ * Faster than extractArtifact — use for non-critical packages where inherited
+ * fields (licenses, SCM) may be missing but throughput matters more.
+ */
+export async function extractArtifactDirect(
+  groupId: string,
+  artifactId: string,
+  version: string,
+  log: (msg: string) => void = () => undefined,
+): Promise<PomExtractionResult> {
+  const purl = `pkg:maven/${groupId}/${artifactId}@${version}`
+  const pom = await fetchPom(groupId, artifactId, version, log)
+
+  if (!pom) {
+    return {
+      groupId,
+      artifactId,
+      version,
+      purl,
+      description: null,
+      licenses: [],
+      licensesRaw: null,
+      scmUrl: null,
+      homepageUrl: null,
+      developers: [],
+      contributors: [],
+      parentHops: 0,
+      error: `POM not found: ${buildPomUrl(groupId, artifactId, version)}`,
+    }
+  }
+
+  const licenses = extractLicenses(pom)
+  const scmUrl = extractStr(pom.scm?.url ?? pom.scm?.connection)
+  const developers = extractPersons(pom.developers?.developer, 'author')
+  const contributors = extractPersons(pom.contributors?.contributor, 'maintainer')
+
+  return {
+    groupId,
+    artifactId,
+    version,
+    purl,
+    description: extractStr(pom.description),
+    licenses,
+    licensesRaw: licenses.length > 0 ? licenses.join(', ') : null,
+    scmUrl,
+    homepageUrl: extractStr(pom.url),
+    developers,
+    contributors,
+    parentHops: 0,
+    error: null,
+  }
+}
+
+/**
+ * Fetches and resolves POM metadata for the given Maven artifact, following
+ * the parent chain to inherit licenses and SCM when not in the direct POM.
  * Always returns a result object; errors are captured in `result.error`.
  */
 export async function extractArtifact(

services/apps/packages_worker/src/pom-fetcher/extract.ts

@@ -1,6 +1,6 @@
 /**
- * Resolves the latest release version of a Maven artifact using the
- * maven-metadata.xml endpoint on Maven Central.
+ * Fetches maven-metadata.xml for a Maven artifact and returns the full version
+ * list plus the current release version.
  *
  * URL format:
  *   https://repo1.maven.org/maven2/{groupPath}/{artifactId}/maven-metadata.xml
@@ -24,14 +24,19 @@ const parser = new XMLParser({
   parseAttributeValue: false,
 })
 
+export interface MavenVersionsMetadata {
+  versions: string[]
+  releaseVersion: string | null
+}
+
 async function sleep(ms: number): Promise<void> {
   return new Promise((r) => setTimeout(r, ms))
 }
 
-export async function resolveLatestVersion(
+export async function resolveVersionsList(
   groupId: string,
   artifactId: string,
-): Promise<string | null> {
+): Promise<MavenVersionsMetadata | null> {
   const groupPath = groupId.replace(/\./g, '/')
   const url = `${MAVEN_REPO}/${groupPath}/${artifactId}/maven-metadata.xml`
 
@@ -45,11 +50,20 @@ export async function resolveLatestVersion(
       const release = typeof versioning?.release === 'string' ? versioning.release.trim() : null
       const latest = typeof versioning?.latest === 'string' ? versioning.latest.trim() : null
 
-      return release || latest || null
+      const rawVersions = versioning?.versions?.version
+      let versions: string[] = []
+      if (Array.isArray(rawVersions)) {
+        versions = rawVersions.map((v: unknown) => String(v).trim()).filter(Boolean)
+      } else if (typeof rawVersions === 'string' && rawVersions.trim()) {
+        versions = [rawVersions.trim()]
+      }
+
+      return { versions, releaseVersion: release || latest || null }
     } catch (err) {
       if (axios.isAxiosError(err)) {
         if (err.response?.status === 404) return null
-        if (err.response?.status === 429 && attempt < MAX_RETRIES) {
+        // 429 = explicit rate limit, 403 = CDN throttle (Maven Central uses both)
+        if ((err.response?.status === 429 || err.response?.status === 403) && attempt < MAX_RETRIES) {
           const delay = RETRY_BASE_MS * 2 ** attempt + Math.random() * 500
           await sleep(delay)
           continue
@@ -61,3 +75,11 @@ export async function resolveLatestVersion(
 
   return null
 }
+
+export async function resolveLatestVersion(
+  groupId: string,
+  artifactId: string,
+): Promise<string | null> {
+  const meta = await resolveVersionsList(groupId, artifactId)
+  return meta?.releaseVersion ?? null
+}