Merge branch 'main' into test-kit-primitives

Author: skwowet

services/apps/packages_worker/package.json

@@ -54,6 +54,8 @@
     "dev:rubygems-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=rubygems-worker SERVICE=rubygems-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9244 src/bin/rubygems-worker.ts",
     "backfill:maven": "SERVICE=maven tsx src/bin/maven-backfill.ts",
     "backfill:maven:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=maven LOG_LEVEL=info tsx src/bin/maven-backfill.ts",
+    "backfill:maven-repo-url": "SERVICE=maven tsx src/bin/maven-repo-url-backfill.ts",
+    "backfill:maven-repo-url:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=maven LOG_LEVEL=info tsx src/bin/maven-repo-url-backfill.ts",
     "import:maven-maintainers": "SERVICE=maven tsx src/maven/scripts/importMaintainersFromCsv.ts",
     "import:maven-maintainers:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=maven tsx src/maven/scripts/importMaintainersFromCsv.ts",
     "import:sonatype-popularity": "SERVICE=maven tsx src/maven/scripts/importSonatypePopularityFromCsv.ts",

services/apps/packages_worker/src/bin/maven-backfill.ts

@@ -2,7 +2,10 @@ import { getServiceLogger } from '@crowd/logging'
 
 import { getMavenConfig } from '../config'
 import { getPackagesDb } from '../db'
-import { runMavenCriticalBackfill } from '../maven/runMavenEnrichmentLoop'
+import {
+  runMavenCriticalBackfill,
+  runMavenCriticalForceBackfill,
+} from '../maven/runMavenEnrichmentLoop'
 
 const log = getServiceLogger()
 
@@ -25,7 +28,14 @@ const main = async () => {
     process.env.MAVEN_FETCHER_BASE_URL_BACKFILL ??
     'https://maven-central.storage-download.googleapis.com/maven2'
 
-  log.info('maven backfill starting (one-shot, full extraction)...')
+  // --force: re-run POM extraction over EVERY critical row, ignoring the
+  // staleness window. Use to fully re-apply extraction changes (e.g. SCM
+  // interpolation) after the queue has already drained. The default path only
+  // picks rows due by refreshDays and cannot be coaxed into a full pass by
+  // setting refreshDays=0 (that reprocesses the first batch forever).
+  const force = process.argv.includes('--force')
+
+  log.info({ force }, 'maven backfill starting (one-shot, full extraction)...')
 
   const config = getMavenConfig()
   log.info(config, 'Config loaded')
@@ -34,7 +44,9 @@ const main = async () => {
   await qx.selectOne('SELECT 1')
   log.info('Connected to packages-db.')
 
-  const totals = await runMavenCriticalBackfill(qx, config, () => shuttingDown)
+  const totals = force
+    ? await runMavenCriticalForceBackfill(qx, config, () => shuttingDown)
+    : await runMavenCriticalBackfill(qx, config, () => shuttingDown)
 
   log.info({ ...totals }, 'maven backfill complete')
   process.exit(0)

services/apps/packages_worker/src/bin/maven-repo-url-backfill.ts

@@ -0,0 +1,61 @@
+import { getServiceLogger } from '@crowd/logging'
+
+import { getPackagesDb } from '../db'
+import { backfillMavenRepositoryUrls } from '../maven/backfillRepositoryUrl'
+
+const log = getServiceLogger()
+
+let shuttingDown = false
+
+// Graceful stop: finish the in-flight batch, then exit. Safe to interrupt — every
+// write is an idempotent UPDATE recomputed from declared_repository_url, so
+// re-running simply reprocesses and skips rows that already match.
+const shutdown = () => {
+  if (shuttingDown) return
+  shuttingDown = true
+  log.info('Shutting down maven repo-url backfill (stopping after the current batch)...')
+}
+
+process.on('SIGINT', shutdown)
+process.on('SIGTERM', shutdown)
+
+const DEFAULT_BATCH_SIZE = 5000
+
+const main = async () => {
+  const dryRun = process.argv.includes('--dry-run')
+  const criticalOnly = process.argv.includes('--critical-only')
+  const rawBatchSize = process.env.MAVEN_REPO_URL_BACKFILL_BATCH_SIZE
+  const parsedBatchSize = rawBatchSize === undefined ? DEFAULT_BATCH_SIZE : Number(rawBatchSize)
+  if (!Number.isInteger(parsedBatchSize) || parsedBatchSize <= 0) {
+    log.error(
+      { MAVEN_REPO_URL_BACKFILL_BATCH_SIZE: rawBatchSize },
+      'MAVEN_REPO_URL_BACKFILL_BATCH_SIZE must be a positive integer',
+    )
+    process.exit(1)
+  }
+  const batchSize = parsedBatchSize
+
+  log.info(
+    { dryRun, criticalOnly, batchSize },
+    'maven repo-url backfill starting (recompute normalizeScmUrl from declared_repository_url, no POM fetch)...',
+  )
+
+  const qx = await getPackagesDb()
+  await qx.selectOne('SELECT 1')
+  log.info('Connected to packages-db.')
+
+  const totals = await backfillMavenRepositoryUrls(qx, {
+    batchSize,
+    dryRun,
+    criticalOnly,
+    isShuttingDown: () => shuttingDown,
+  })
+
+  log.info({ ...totals, dryRun }, 'maven repo-url backfill complete')
+  process.exit(0)
+}
+
+main().catch((err) => {
+  log.error({ err }, 'maven repo-url backfill fatal error')
+  process.exit(1)
+})