refactor: more solid and simple structure

Signed-off-by: Umberto Sgueglia <usgueglia@contractor.linuxfoundation.org>

Author: ulemons

backend/.env.dist.local

@@ -199,34 +199,14 @@ OSV_ECOSYSTEMS=npm,Maven
 OSV_TMP_DIR=/tmp/osv
 OSV_BATCH_SIZE=500
 OSV_DERIVE_BATCH_SIZE=1000
-# pom-fetcher non-critical (DB-only)
-POM_FETCHER_NON_CRITICAL_BATCH_SIZE=500
-POM_FETCHER_NON_CRITICAL_CONCURRENCY=20
-
-# pom-fetcher critical (HTTP)
-POM_FETCHER_BATCH_SIZE=100
-POM_FETCHER_CONCURRENCY=5
-# ── non-critical DB-only (usato quando DIRECT_POM_FOR_ALL=false) ──────────────
-# POM_FETCHER_NON_CRITICAL_BATCH_SIZE=500
-POM_FETCHER_NON_CRITICAL_CONCURRENCY=20    # solo DB writes, nessun HTTP
-POM_FETCHER_NON_CRITICAL_REFRESH_DAYS=1
-
-# ── non-critical HTTP / direct-pom (usato quando DIRECT_POM_FOR_ALL=true) ─────
-# POM_FETCHER_NON_CRITICAL_POM_CONCURRENCY=5 
-# POM_FETCHER_NON_CRITICAL_POM_REFRESH_DAYS=1
-
-# ── critical HTTP / full-pom (sempre attivo) ──────────────────────────────────
-# POM_FETCHER_BATCH_SIZE=100
-# POM_FETCHER_CONCURRENCY=5
-# POM_FETCHER_FULL_REFRESH_DAYS=1
-
-# ── modalità e rate limiting ──────────────────────────────────────────────────
-# POM_FETCHER_DIRECT_POM_FOR_ALL=true
-# POM_FETCHER_GROUP_DELAY_MS=200
-
-
+# maven enricher
 POM_FETCHER_BATCH_SIZE=50
 POM_FETCHER_CONCURRENCY=3
+POM_FETCHER_NON_CRITICAL_BATCH_SIZE=500
+POM_FETCHER_NON_CRITICAL_CONCURRENCY=20
 POM_FETCHER_REFRESH_DAYS=1
 POM_FETCHER_GROUP_DELAY_MS=500
 POM_FETCHER_IDLE_SLEEP_SEC=3600
+# Set to 'true' on first run against a fresh/restored DB to skip the version-unchanged
+# optimisation and force full POM extraction. Set to 'false' after the first pass.
+POM_FETCHER_FORCE_FULL_EXTRACTION=false

scripts/services/pom-fetcher.yaml scripts/services/maven.yamlscripts/services/pom-fetcher.yaml renamed to scripts/services/maven.yaml

@@ -3,7 +3,7 @@ version: '3.1'
 x-env-args: &env-args
   DOCKER_BUILDKIT: 1
   NODE_ENV: docker
-  SERVICE: pom-fetcher
+  SERVICE: maven
   SHELL: /bin/sh
   SUPPRESS_NO_CONFIG_WARNING: 'true'
   LOG_LEVEL: 'info'
@@ -14,11 +14,11 @@ x-env-args: &env-args
   POM_FETCHER_IDLE_SLEEP_SEC: '3600'
 
 services:
-  pom-fetcher:
+  maven:
     build:
       context: ../../
       dockerfile: ./scripts/services/docker/Dockerfile.packages-worker
-    command: 'pnpm run start:pom-fetcher'
+    command: 'pnpm run start:maven'
     working_dir: /usr/crowd/app/services/apps/packages_worker
     env_file:
       - ../../backend/.env.dist.local
@@ -31,11 +31,11 @@ services:
     networks:
       - crowd-bridge
 
-  pom-fetcher-dev:
+  maven-dev:
     build:
       context: ../../
       dockerfile: ./scripts/services/docker/Dockerfile.packages-worker
-    command: 'pnpm run dev:pom-fetcher'
+    command: 'pnpm run dev:maven'
     working_dir: /usr/crowd/app/services/apps/packages_worker
     # user: '${USER_ID}:${GROUP_ID}'
     env_file:
@@ -45,7 +45,7 @@ services:
       - ../../backend/.env.override.composed
     environment:
       <<: *env-args
-    hostname: pom-fetcher
+    hostname: maven
     networks:
       - crowd-bridge
     volumes:

services/apps/packages_worker/package.json

@@ -5,10 +5,11 @@
     "start:packages-worker": "CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=packages-worker tsx src/bin/packages-worker.ts",
     "start:deps-dev-ingest": "CROWD_TEMPORAL_TASKQUEUE=deps-dev-ingest CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=deps-dev-ingest tsx src/bin/deps-dev-ingest.ts",
     "start:github-repos-enricher": "SERVICE=github-repos-enricher tsx src/bin/github-repos-enricher.ts",
-    "start:pom-fetcher": "SERVICE=pom-fetcher tsx src/bin/pom-fetcher.ts",
+    "start:maven": "SERVICE=maven tsx src/bin/maven.ts",
     "dev:packages-worker": "CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=packages-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9233 src/bin/packages-worker.ts",
     "dev:deps-dev-ingest": "CROWD_TEMPORAL_TASKQUEUE=deps-dev-ingest CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=deps-dev-ingest nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9235 src/bin/deps-dev-ingest.ts",
     "dev:github-repos-enricher": "SERVICE=github-repos-enricher LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9234 src/bin/github-repos-enricher.ts",
+    "dev:maven": "SERVICE=maven LOG_LEVEL=info nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9235 src/bin/maven.ts",
     "dev:packages-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=packages-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9233 src/bin/packages-worker.ts",
     "dev:deps-dev-ingest:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=deps-dev-ingest CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=deps-dev-ingest nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9235 src/bin/deps-dev-ingest.ts",
     "dev:github-repos-enricher:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=github-repos-enricher LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9234 src/bin/github-repos-enricher.ts",
@@ -18,8 +19,7 @@
     "export-to-bucket": "SERVICE=deps-dev-ingest tsx src/scripts/exportToBucket.ts",
     "export-to-bucket:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=deps-dev-ingest tsx src/scripts/exportToBucket.ts",
     "monitor:osspckgs:local": "bash -c 'set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && node ../../../scripts/monitor-osspckgs.mjs'",
-    "dev:pom-fetcher": "SERVICE=pom-fetcher LOG_LEVEL=info nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9235 src/bin/pom-fetcher.ts",
-    "dev:pom-fetcher:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=pom-fetcher LOG_LEVEL=info nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9235 src/bin/pom-fetcher.ts",
+    "dev:maven:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=maven LOG_LEVEL=info nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9235 src/bin/maven.ts",
     "lint": "npx eslint --ext .ts src --max-warnings=0",
     "format": "npx prettier --write \"src/**/*.ts\"",
     "format-check": "npx prettier --check .",

services/apps/packages_worker/src/activities.ts

@@ -1,4 +1,4 @@
 export * from './deps-dev/activities'
 export * from './npm/activities'
 export { osvSyncEcosystem, osvDeriveCriticalFlag } from './osv/activities'
-export { processMavenCriticalBatch, processMavenNonCriticalBatch } from './pom-fetcher/activities'
+export { processMavenCriticalBatch, processMavenNonCriticalBatch } from './maven/activities'

…s/packages_worker/src/bin/pom-fetcher.ts …es/apps/packages_worker/src/bin/maven.tsservices/apps/packages_worker/src/bin/pom-fetcher.ts renamed to services/apps/packages_worker/src/bin/maven.ts services/apps/packages_worker/src/bin/pom-fetcher.ts renamed to services/apps/packages_worker/src/bin/maven.ts

@@ -1,8 +1,8 @@
 import { getServiceLogger } from '@crowd/logging'
 
-import { getPomFetcherConfig } from '../config'
+import { getMavenConfig } from '../config'
 import { getPackagesDb } from '../db'
-import { runPomEnrichmentLoop } from '../pom-fetcher/runPomEnrichmentLoop'
+import { runMavenEnrichmentLoop } from '../maven/runMavenEnrichmentLoop'
 
 const log = getServiceLogger()
 
@@ -11,29 +11,29 @@ let shuttingDown = false
 const shutdown = async () => {
   if (shuttingDown) return
   shuttingDown = true
-  log.info('Shutting down pom-fetcher...')
+  log.info('Shutting down maven...')
 }
 
 process.on('SIGINT', shutdown)
 process.on('SIGTERM', shutdown)
 
 const main = async () => {
-  log.info('pom-fetcher starting...')
+  log.info('maven starting...')
 
-  const config = getPomFetcherConfig()
+  const config = getMavenConfig()
   log.info(config, 'Config loaded')
 
   const qx = await getPackagesDb()
   await qx.selectOne('SELECT 1')
   log.info('Connected to packages-db.')
 
-  await runPomEnrichmentLoop(qx, config, () => shuttingDown)
+  await runMavenEnrichmentLoop(qx, config, () => shuttingDown)
 
-  log.info('pom-fetcher stopped.')
+  log.info('maven stopped.')
   process.exit(0)
 }
 
 main().catch((err) => {
-  log.error({ err }, 'pom-fetcher fatal error')
+  log.error({ err }, 'maven fatal error')
   process.exit(1)
 })

services/apps/packages_worker/src/bin/packages-worker.ts

@@ -1,12 +1,13 @@
 import { scheduleNpmIngest } from '../npm/schedule'
 import { scheduleOsvSync } from '../osv/schedule'
-import { schedulePomFetcher } from '../pom-fetcher/schedule'
+import { scheduleMavenCritical, scheduleMavenNonCritical } from '../maven/schedule'
 import { svc } from '../service'
 
 setImmediate(async () => {
   await svc.init()
   await scheduleNpmIngest()
   await scheduleOsvSync()
-  await schedulePomFetcher()
+  await scheduleMavenCritical()
+  await scheduleMavenNonCritical()
   await svc.start()
 })

services/apps/packages_worker/src/config.ts

@@ -33,12 +33,15 @@ export function getEnricherConfig() {
   }
 }
 
-export function getPomFetcherConfig() {
+export function getMavenConfig() {
   return {
-    batchSize: parseInt(process.env.POM_FETCHER_BATCH_SIZE ?? '50', 10),
-    concurrency: parseInt(process.env.POM_FETCHER_CONCURRENCY ?? '5', 10),
-    refreshDays: parseInt(process.env.POM_FETCHER_REFRESH_DAYS ?? '1', 10),
-    groupDelayMs: parseInt(process.env.POM_FETCHER_GROUP_DELAY_MS ?? '200', 10),
-    idleSleepSec: parseInt(process.env.POM_FETCHER_IDLE_SLEEP_SEC ?? '3600', 10),
+    batchSize: requireEnvInt('POM_FETCHER_BATCH_SIZE'),
+    concurrency: requireEnvInt('POM_FETCHER_CONCURRENCY'),
+    nonCriticalBatchSize: requireEnvInt('POM_FETCHER_NON_CRITICAL_BATCH_SIZE'),
+    nonCriticalConcurrency: requireEnvInt('POM_FETCHER_NON_CRITICAL_CONCURRENCY'),
+    refreshDays: requireEnvInt('POM_FETCHER_REFRESH_DAYS'),
+    groupDelayMs: requireEnvInt('POM_FETCHER_GROUP_DELAY_MS'),
+    idleSleepSec: requireEnvInt('POM_FETCHER_IDLE_SLEEP_SEC'),
+    forceFullExtraction: requireEnv('POM_FETCHER_FORCE_FULL_EXTRACTION') === 'true',
   }
 }

…ackages_worker/src/pom-fetcher/README.md …apps/packages_worker/src/maven/README.mdservices/apps/packages_worker/src/pom-fetcher/README.md renamed to services/apps/packages_worker/src/maven/README.md services/apps/packages_worker/src/pom-fetcher/README.md renamed to services/apps/packages_worker/src/maven/README.md

@@ -127,12 +127,12 @@ The POM fetcher seeds `repos` with URL-derivable fields only. The GitHub enriche
 
 | Value | Meaning |
 |-------|---------|
-| `pom_fetcher` | Critical — full POM + parent resolution succeeded |
-| `pom_fetcher_direct` | Non-critical — direct POM fetch succeeded (no parent resolution) |
-| `pom_fetcher_not_on_central` | `maven-metadata.xml` not found on `repo1.maven.org` — artifact is hosted on a third-party repository (e.g. WSO2 Nexus, JBoss, Atlassian). Universe data came from an aggregator (deps.dev, OSV). |
-| `pom_fetcher_no_version` | `maven-metadata.xml` found but `<release>` is empty — artifact has no stable release |
-| `pom_fetcher_error` | `maven-metadata.xml` has a release version but the `.pom` file for that version is a 404. Typical cause: partial deploy to Maven Central (metadata updated, artifact not uploaded) or Eclipse P2 feature artifacts that don't publish a standard POM. |
-| `pom_fetcher_rate_limited` | Maven Central returned 403/429 on all retry attempts. Package will be retried on the next pass. |
+| `maven` | Critical — full POM + parent resolution succeeded |
+| `maven_direct` | Non-critical — direct POM fetch succeeded (no parent resolution) |
+| `maven_not_on_central` | `maven-metadata.xml` not found on `repo1.maven.org` — artifact is hosted on a third-party repository (e.g. WSO2 Nexus, JBoss, Atlassian). Universe data came from an aggregator (deps.dev, OSV). |
+| `maven_no_version` | `maven-metadata.xml` found but `<release>` is empty — artifact has no stable release |
+| `maven_error` | `maven-metadata.xml` has a release version but the `.pom` file for that version is a 404. Typical cause: partial deploy to Maven Central (metadata updated, artifact not uploaded) or Eclipse P2 feature artifacts that don't publish a standard POM. |
+| `maven_rate_limited` | Maven Central returned 403/429 on all retry attempts. Package will be retried on the next pass. |
 | `packages_universe` | Non-critical, DB-only mode (`POM_FETCHER_DIRECT_POM_FOR_ALL=false`) — only universe stats copied |
 
 ---
@@ -143,19 +143,19 @@ The POM fetcher seeds `repos` with URL-derivable fields only. The GitHub enriche
 
 WSO2 publishes some artifacts exclusively to their own Nexus at `maven.wso2.org`. A subset of their artifacts appear in `packages_universe` (sourced from deps.dev/OSV which aggregates all Maven repositories) but are **not** available on `repo1.maven.org/maven2`.
 
-Affected pattern: `org.wso2.carbon.*` — specifically `.feature` Eclipse P2 artifacts and `.stub` artifacts. These are written with `ingestion_source = 'pom_fetcher_not_on_central'` and are not retried until the next `nonCriticalPomRefreshDays` window.
+Affected pattern: `org.wso2.carbon.*` — specifically `.feature` Eclipse P2 artifacts and `.stub` artifacts. These are written with `ingestion_source = 'maven_not_on_central'` and are not retried until the next `nonCriticalPomRefreshDays` window.
 
 ### Eclipse P2 Feature Artifacts (`*.feature`)
 
-Eclipse/OSGi feature artifacts (e.g. `org.wso2.carbon.identity.xacml.server.feature`) are packaged as `.zip` files, not `.jar`. Some publishers update `maven-metadata.xml` on Central without uploading the corresponding `.pom`. These land in `pom_fetcher_error`. No fix is possible without the publisher correcting their CI/CD pipeline.
+Eclipse/OSGi feature artifacts (e.g. `org.wso2.carbon.identity.xacml.server.feature`) are packaged as `.zip` files, not `.jar`. Some publishers update `maven-metadata.xml` on Central without uploading the corresponding `.pom`. These land in `maven_error`. No fix is possible without the publisher correcting their CI/CD pipeline.
 
 ### Maven Central 403 rate limiting
 
 Maven Central (`repo1.maven.org`) restituisce 403 come meccanismo di throttle oltre al canonico 429. Il comportamento è gestito a due livelli:
 
 1. **Retry con backoff esponenziale** — 403 e 429 vengono ritentati fino a 3 volte (2s base, ×2 per tentativo). Gestito in `getWithRetry` (extract.ts) e `resolveVersionsList` (metadata.ts).
 
-2. **Fallback su DB** — se tutti i retry esauriscono, il pacchetto viene scritto con `ingestion_source = 'pom_fetcher_rate_limited'` e `last_synced_at = NOW()`, evitando loop infiniti. Verrà ritentato al prossimo ciclo di refresh.
+2. **Fallback su DB** — se tutti i retry esauriscono, il pacchetto viene scritto con `ingestion_source = 'maven_rate_limited'` e `last_synced_at = NOW()`, evitando loop infiniti. Verrà ritentato al prossimo ciclo di refresh.
 
 **Causa root dei 403 persistenti:** `packages_universe` è ordinato per `rank_in_ecosystem`, quindi pacchetti dello stesso namespace (es. `com.google.apis`, `org.wso2`, `software.amazon.awssdk`) si raggruppano nel batch e colpiscono lo stesso CDN node di Maven Central in rapida successione. Il rate limit scatta sistematicamente dopo ~150–200 pacchetti processati.
 
@@ -167,7 +167,7 @@ Namespace noti per triggerare il rate limit a causa dell'alta densità di artefa
 
 ### Partial Maven Central Deploys
 
-Occasionally a publisher's CI/CD updates `<release>` in `maven-metadata.xml` before the `.pom` is fully propagated to all Central mirrors. These appear as `pom_fetcher_error` on the first pass and usually resolve on the next periodic refresh.
+Occasionally a publisher's CI/CD updates `<release>` in `maven-metadata.xml` before the `.pom` is fully propagated to all Central mirrors. These appear as `maven_error` on the first pass and usually resolve on the next periodic refresh.
 
 ---
 
@@ -221,7 +221,7 @@ The daily Temporal schedule has a **12-hour workflow timeout**. With `POM_FETCHE
 
 ## Scheduling
 
-Runs daily at **4am UTC** via Temporal schedule `maven-pom-fetcher`.
+Runs daily at **4am UTC** via Temporal schedule `maven`.
 Overlap policy: `SKIP` — if a previous run is still active, the new trigger is dropped.
 Catchup window: 1 hour.
 Workflow timeout: 12 hours.
@@ -274,9 +274,9 @@ ORDER BY packages DESC;
 ```
 
 Expected behaviour:
-- `pom_fetcher` (critical, full resolution) → high repo coverage
-- `pom_fetcher_direct` (non-critical, no parent resolution) → low repo coverage
-- `pom_fetcher_not_on_central` / `pom_fetcher_error` → no repo (no POM data)
+- `maven` (critical, full resolution) → high repo coverage
+- `maven_direct` (non-critical, no parent resolution) → low repo coverage
+- `maven_not_on_central` / `maven_error` → no repo (no POM data)
 
 Repo coverage will grow naturally as the critical package set expands and as non-critical packages hit their 30-day POM refresh window.
 

…ges_worker/src/pom-fetcher/activities.ts …/packages_worker/src/maven/activities.tsservices/apps/packages_worker/src/pom-fetcher/activities.ts renamed to services/apps/packages_worker/src/maven/activities.ts services/apps/packages_worker/src/pom-fetcher/activities.ts renamed to services/apps/packages_worker/src/maven/activities.ts

@@ -1,23 +1,23 @@
 import { getServiceChildLogger } from '@crowd/logging'
 
-import { getPomFetcherConfig } from '../config'
+import { getMavenConfig } from '../config'
 import { getPackagesDb } from '../db'
-import { BatchResult, processBatch } from './runPomEnrichmentLoop'
+import { BatchResult, processBatch } from './runMavenEnrichmentLoop'
 
-const log = getServiceChildLogger('pom-fetcher-activity')
+const log = getServiceChildLogger('maven-activity')
 
 export async function processMavenCriticalBatch(): Promise<BatchResult> {
-  const config = getPomFetcherConfig()
+  const config = getMavenConfig()
   const qx = await getPackagesDb()
   const result = await processBatch(qx, config, true)
-  log.info({ processed: result.processed, skipped: result.skipped, errors: result.errors }, 'Maven critical batch complete')
+  log.info({ processed: result.processed, skipped: result.skipped, unchanged: result.unchanged, error: result.error }, 'Maven critical batch complete')
   return result
 }
 
 export async function processMavenNonCriticalBatch(): Promise<BatchResult> {
-  const config = getPomFetcherConfig()
+  const config = getMavenConfig()
   const qx = await getPackagesDb()
   const result = await processBatch(qx, config, false)
-  log.info({ processed: result.processed, skipped: result.skipped, errors: result.errors }, 'Maven non-critical batch complete')
+  log.info({ processed: result.processed, skipped: result.skipped, unchanged: result.unchanged, error: result.error }, 'Maven non-critical batch complete')
   return result
 }