feat: pom extractor

Signed-off-by: Umberto Sgueglia <usgueglia@contractor.linuxfoundation.org>

Author: ulemons

pnpm-lock.yaml

@@ -0,0 +1,69 @@
+version: '3.1'
+
+x-env-args: &env-args
+  DOCKER_BUILDKIT: 1
+  NODE_ENV: docker
+  SERVICE: pom-fetcher
+  SHELL: /bin/sh
+  SUPPRESS_NO_CONFIG_WARNING: 'true'
+  POM_FETCHER_BATCH_SIZE: '200'
+  POM_FETCHER_CONCURRENCY: '10'
+  POM_FETCHER_STALE_DAYS: '7'
+  POM_FETCHER_IDLE_SLEEP_SEC: '3600'
+
+services:
+  pom-fetcher:
+    build:
+      context: ../../
+      dockerfile: ./scripts/services/docker/Dockerfile.packages-worker
+    command: 'pnpm run start:pom-fetcher'
+    working_dir: /usr/crowd/app/services/apps/packages_worker
+    env_file:
+      - ../../backend/.env.dist.local
+      - ../../backend/.env.dist.composed
+      - ../../backend/.env.override.local
+      - ../../backend/.env.override.composed
+    environment:
+      <<: *env-args
+    restart: always
+    networks:
+      - crowd-bridge
+
+  pom-fetcher-dev:
+    build:
+      context: ../../
+      dockerfile: ./scripts/services/docker/Dockerfile.packages-worker
+    command: 'pnpm run dev:pom-fetcher'
+    working_dir: /usr/crowd/app/services/apps/packages_worker
+    # user: '${USER_ID}:${GROUP_ID}'
+    env_file:
+      - ../../backend/.env.dist.local
+      - ../../backend/.env.dist.composed
+      - ../../backend/.env.override.local
+      - ../../backend/.env.override.composed
+    environment:
+      <<: *env-args
+    hostname: pom-fetcher
+    networks:
+      - crowd-bridge
+    volumes:
+      - ../../services/libs/audit-logs/src:/usr/crowd/app/services/libs/audit-logs/src
+      - ../../services/libs/common/src:/usr/crowd/app/services/libs/common/src
+      - ../../services/libs/common_services/src:/usr/crowd/app/services/libs/common_services/src
+      - ../../services/libs/data-access-layer/src:/usr/crowd/app/services/libs/data-access-layer/src
+      - ../../services/libs/database/src:/usr/crowd/app/services/libs/database/src
+      - ../../services/libs/integrations/src:/usr/crowd/app/services/libs/integrations/src
+      - ../../services/libs/logging/src:/usr/crowd/app/services/libs/logging/src
+      - ../../services/libs/nango/src:/usr/crowd/app/services/libs/nango/src
+      - ../../services/libs/opensearch/src:/usr/crowd/app/services/libs/opensearch/src
+      - ../../services/libs/queue/src:/usr/crowd/app/services/libs/queue/src
+      - ../../services/libs/redis/src:/usr/crowd/app/services/libs/redis/src
+      - ../../services/libs/snowflake/src:/usr/crowd/app/services/libs/snowflake/src
+      - ../../services/libs/telemetry/src:/usr/crowd/app/services/libs/telemetry/src
+      - ../../services/libs/temporal/src:/usr/crowd/app/services/libs/temporal/src
+      - ../../services/libs/types/src:/usr/crowd/app/services/libs/types/src
+      - ../../services/apps/packages_worker/src:/usr/crowd/app/services/apps/packages_worker/src
+
+networks:
+  crowd-bridge:
+    external: true

scripts/services/pom-fetcher.yaml

@@ -0,0 +1,42 @@
+import { getServiceLogger } from '@crowd/logging'
+
+import { getPomFetcherConfig } from '../config'
+import { getPackagesDb } from '../db'
+import { runPomEnrichmentLoop } from '../pom-fetcher/runPomEnrichmentLoop'
+
+const log = getServiceLogger()
+
+let shuttingDown = false
+
+const shutdown = async () => {
+  if (shuttingDown) return
+  shuttingDown = true
+  log.info('Shutting down pom-fetcher...')
+}
+
+process.on('SIGINT', shutdown)
+process.on('SIGTERM', shutdown)
+
+const main = async () => {
+  log.info('pom-fetcher starting...')
+
+  const config = getPomFetcherConfig()
+  log.info(
+    { batchSize: config.batchSize, concurrency: config.concurrency, staleDays: config.staleDays },
+    'Config loaded',
+  )
+
+  const qx = await getPackagesDb()
+  await qx.selectOne('SELECT 1')
+  log.info('Connected to packages-db.')
+
+  await runPomEnrichmentLoop(qx, config, () => shuttingDown)
+
+  log.info('pom-fetcher stopped.')
+  process.exit(0)
+}
+
+main().catch((err) => {
+  log.error({ err }, 'pom-fetcher fatal error')
+  process.exit(1)
+})

services/apps/packages_worker/src/bin/pom-fetcher.ts

@@ -32,3 +32,12 @@ export function getEnricherConfig() {
     idleSleepSec: requireEnvInt('ENRICHER_IDLE_SLEEP_SEC'),
   }
 }
+
+export function getPomFetcherConfig() {
+  return {
+    batchSize: parseInt(process.env.POM_FETCHER_BATCH_SIZE ?? '200', 10),
+    concurrency: parseInt(process.env.POM_FETCHER_CONCURRENCY ?? '10', 10),
+    staleDays: parseInt(process.env.POM_FETCHER_STALE_DAYS ?? '7', 10),
+    idleSleepSec: parseInt(process.env.POM_FETCHER_IDLE_SLEEP_SEC ?? '3600', 10),
+  }
+}