feat: pom fetcher (CM-1210) (#4179)

Signed-off-by: Mouad BANI <mouad-mb@outlook.com>
Signed-off-by: Umberto Sgueglia <usgueglia@contractor.linuxfoundation.org>
Co-authored-by: Mouad BANI <mouad-mb@outlook.com>

Author: ulemons

backend/.env.dist.local

@@ -185,9 +185,9 @@ ENRICHER_BATCH_SIZE=100
 ENRICHER_REPO_UPDATE_INTERVAL_HOURS=24
 ENRICHER_IDLE_SLEEP_SEC=60
 
-OSSPCKGS_GCP_PROJECT=
-OSSPCKGS_GCS_BUCKET=
-OSSPCKGS_GCP_CREDENTIALS_B64=
+OSSPCKGS_GCP_PROJECT=local-dev
+OSSPCKGS_GCS_BUCKET=local-dev
+OSSPCKGS_GCP_CREDENTIALS_B64=e30=
 
 # osv-sync (Temporal-scheduled; see services/apps/packages_worker/src/osv/schedule.ts)
 # OSV_ECOSYSTEMS uses OSV's canonical bucket case (npm lowercase, Maven titlecase) because
@@ -199,3 +199,14 @@ OSV_ECOSYSTEMS=npm,Maven
 OSV_TMP_DIR=/tmp/osv
 OSV_BATCH_SIZE=500
 OSV_DERIVE_BATCH_SIZE=1000
+
+# maven enricher
+
+MAVEN_FETCHER_BATCH_SIZE=2000
+MAVEN_FETCHER_CONCURRENCY=10
+MAVEN_FETCHER_NON_CRITICAL_BATCH_SIZE=500
+MAVEN_FETCHER_NON_CRITICAL_CONCURRENCY=20
+MAVEN_FETCHER_REFRESH_DAYS=1
+MAVEN_FETCHER_GROUP_DELAY_MS=100
+MAVEN_FETCHER_BASE_URL_BACKFILL=https://maven-central.storage-download.googleapis.com/maven2
+MAVEN_FETCHER_BASE_URL_INCREMENTAL=https://repo1.maven.org/maven2

pnpm-lock.yaml

@@ -1,4 +1,4 @@
 DOCKERFILE="./services/docker/Dockerfile.packages"
 CONTEXT="../"
 REPO="sjc.ocir.io/axbydjxa5zuh/packages"
-SERVICES="packages github-repos-enricher deps-dev-ingest npm-worker"
+SERVICES="packages github-repos-enricher deps-dev-ingest npm-worker maven-worker"

scripts/builders/packages.env

@@ -0,0 +1,67 @@
+version: '3.1'
+
+x-env-args: &env-args
+  DOCKER_BUILDKIT: 1
+  NODE_ENV: docker
+  SERVICE: maven-worker
+  CROWD_TEMPORAL_TASKQUEUE: packages-worker
+  CROWD_TEMPORAL_NAMESPACE: ${CROWD_PACKAGES_TEMPORAL_NAMESPACE}
+  SHELL: /bin/sh
+  SUPPRESS_NO_CONFIG_WARNING: 'true'
+
+services:
+  maven-worker:
+    build:
+      context: ../../
+      dockerfile: ./scripts/services/docker/Dockerfile.packages
+    command: 'pnpm run start:maven-worker'
+    working_dir: /usr/crowd/app/services/apps/packages_worker
+    env_file:
+      - ../../backend/.env.dist.local
+      - ../../backend/.env.dist.composed
+      - ../../backend/.env.override.local
+      - ../../backend/.env.override.composed
+    environment:
+      <<: *env-args
+    restart: always
+    networks:
+      - crowd-bridge
+
+  maven-worker-dev:
+    build:
+      context: ../../
+      dockerfile: ./scripts/services/docker/Dockerfile.packages
+    command: 'pnpm run dev:maven-worker'
+    working_dir: /usr/crowd/app/services/apps/packages_worker
+    # user: '${USER_ID}:${GROUP_ID}'
+    env_file:
+      - ../../backend/.env.dist.local
+      - ../../backend/.env.dist.composed
+      - ../../backend/.env.override.local
+      - ../../backend/.env.override.composed
+    environment:
+      <<: *env-args
+    hostname: maven-worker
+    networks:
+      - crowd-bridge
+    volumes:
+      - ../../services/libs/audit-logs/src:/usr/crowd/app/services/libs/audit-logs/src
+      - ../../services/libs/common/src:/usr/crowd/app/services/libs/common/src
+      - ../../services/libs/common_services/src:/usr/crowd/app/services/libs/common_services/src
+      - ../../services/libs/data-access-layer/src:/usr/crowd/app/services/libs/data-access-layer/src
+      - ../../services/libs/database/src:/usr/crowd/app/services/libs/database/src
+      - ../../services/libs/integrations/src:/usr/crowd/app/services/libs/integrations/src
+      - ../../services/libs/logging/src:/usr/crowd/app/services/libs/logging/src
+      - ../../services/libs/nango/src:/usr/crowd/app/services/libs/nango/src
+      - ../../services/libs/opensearch/src:/usr/crowd/app/services/libs/opensearch/src
+      - ../../services/libs/queue/src:/usr/crowd/app/services/libs/queue/src
+      - ../../services/libs/redis/src:/usr/crowd/app/services/libs/redis/src
+      - ../../services/libs/snowflake/src:/usr/crowd/app/services/libs/snowflake/src
+      - ../../services/libs/telemetry/src:/usr/crowd/app/services/libs/telemetry/src
+      - ../../services/libs/temporal/src:/usr/crowd/app/services/libs/temporal/src
+      - ../../services/libs/types/src:/usr/crowd/app/services/libs/types/src
+      - ../../services/apps/packages_worker/src:/usr/crowd/app/services/apps/packages_worker/src
+
+networks:
+  crowd-bridge:
+    external: true