feat: fix maven repo gap

Signed-off-by: Umberto Sgueglia <usgueglia@contractor.linuxfoundation.org>

Author: ulemons

services/apps/packages_worker/package.json

@@ -54,6 +54,8 @@
     "dev:rubygems-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=rubygems-worker SERVICE=rubygems-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9244 src/bin/rubygems-worker.ts",
     "backfill:maven": "SERVICE=maven tsx src/bin/maven-backfill.ts",
     "backfill:maven:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=maven LOG_LEVEL=info tsx src/bin/maven-backfill.ts",
+    "backfill:maven-repo-url": "SERVICE=maven tsx src/bin/maven-repo-url-backfill.ts",
+    "backfill:maven-repo-url:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=maven LOG_LEVEL=info tsx src/bin/maven-repo-url-backfill.ts",
     "import:maven-maintainers": "SERVICE=maven tsx src/maven/scripts/importMaintainersFromCsv.ts",
     "import:maven-maintainers:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=maven tsx src/maven/scripts/importMaintainersFromCsv.ts",
     "import:sonatype-popularity": "SERVICE=maven tsx src/maven/scripts/importSonatypePopularityFromCsv.ts",

services/apps/packages_worker/src/bin/maven-repo-url-backfill.ts

@@ -0,0 +1,59 @@
+import { getServiceLogger } from '@crowd/logging'
+
+import { getPackagesDb } from '../db'
+import { backfillMavenRepositoryUrls } from '../maven/backfillRepositoryUrl'
+
+const log = getServiceLogger()
+
+let shuttingDown = false
+
+// Graceful stop: finish the in-flight batch, then exit. Safe to interrupt — every
+// write is an idempotent UPDATE recomputed from declared_repository_url, so
+// re-running simply reprocesses and skips rows that already match.
+const shutdown = () => {
+  if (shuttingDown) return
+  shuttingDown = true
+  log.info('Shutting down maven repo-url backfill (stopping after the current batch)...')
+}
+
+process.on('SIGINT', shutdown)
+process.on('SIGTERM', shutdown)
+
+const DEFAULT_BATCH_SIZE = 5000
+
+const main = async () => {
+  const dryRun = process.argv.includes('--dry-run')
+  const rawBatchSize = process.env.MAVEN_REPO_URL_BACKFILL_BATCH_SIZE
+  const parsedBatchSize = rawBatchSize === undefined ? DEFAULT_BATCH_SIZE : Number(rawBatchSize)
+  if (!Number.isInteger(parsedBatchSize) || parsedBatchSize <= 0) {
+    log.error(
+      { MAVEN_REPO_URL_BACKFILL_BATCH_SIZE: rawBatchSize },
+      'MAVEN_REPO_URL_BACKFILL_BATCH_SIZE must be a positive integer',
+    )
+    process.exit(1)
+  }
+  const batchSize = parsedBatchSize
+
+  log.info(
+    { dryRun, batchSize },
+    'maven repo-url backfill starting (recompute normalizeScmUrl from declared_repository_url, no POM fetch)...',
+  )
+
+  const qx = await getPackagesDb()
+  await qx.selectOne('SELECT 1')
+  log.info('Connected to packages-db.')
+
+  const totals = await backfillMavenRepositoryUrls(qx, {
+    batchSize,
+    dryRun,
+    isShuttingDown: () => shuttingDown,
+  })
+
+  log.info({ ...totals, dryRun }, 'maven repo-url backfill complete')
+  process.exit(0)
+}
+
+main().catch((err) => {
+  log.error({ err }, 'maven repo-url backfill fatal error')
+  process.exit(1)
+})

services/apps/packages_worker/src/maven/__tests__/normalize.test.ts

@@ -163,4 +163,35 @@ describe('normalizeScmUrl', () => {
   it('returns null for non-https result', () => {
     expect(normalizeScmUrl('svn://svn.apache.org/repos/commons-lang')).toBeNull()
   })
+
+  // Gap B — recover repository_url from inputs that were previously dropped
+  it('recovers scm:git: without a scheme', () => {
+    expect(normalizeScmUrl('scm:git:github.com/lum-ai/nxmlreader')).toBe(
+      'https://github.com/lum-ai/nxmlreader',
+    )
+  })
+
+  it('recovers a bare host/owner/repo without a scheme', () => {
+    expect(normalizeScmUrl('github.com/agiledigital/kamon-play-extensions')).toBe(
+      'https://github.com/agiledigital/kamon-play-extensions',
+    )
+  })
+
+  it('upgrades http to https and lower-cases the github path', () => {
+    expect(normalizeScmUrl('http://github.com/kevemueller/kTLSH/tree/master')).toBe(
+      'https://github.com/kevemueller/ktlsh',
+    )
+  })
+
+  // Gap C — reject non-repository URLs so they are never stored
+  it('returns null for website-only URLs', () => {
+    expect(normalizeScmUrl('https://meson.ai/')).toBeNull()
+    expect(normalizeScmUrl('http://source.android.com')).toBeNull()
+  })
+
+  it('returns null for placeholders and free-form text', () => {
+    expect(normalizeScmUrl('Private')).toBeNull()
+    expect(normalizeScmUrl('${scm-url}')).toBeNull()
+    expect(normalizeScmUrl('http://cvs.sourceforge.net/cgi-bin/viewcvs.cgi/foo')).toBeNull()
+  })
 })

services/apps/packages_worker/src/maven/backfillRepositoryUrl.ts

@@ -0,0 +1,76 @@
+import {
+  listMavenPackagesForRepoUrlRecompute,
+  updateMavenRepositoryUrls,
+} from '@crowd/data-access-layer'
+import { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor'
+import { getServiceChildLogger } from '@crowd/logging'
+
+import { normalizeScmUrl } from './extract'
+
+const log = getServiceChildLogger('maven-repo-url-backfill')
+
+export type RepoUrlBackfillTotals = {
+  scanned: number
+  filled: number // Gap B: NULL → canonical value
+  cleared: number // Gap C: non-repo value → NULL
+  rewritten: number // non-canonical value → different canonical value
+  unchanged: number
+}
+
+/**
+ * Recomputes `repository_url` for every Maven row directly from the stored
+ * `declared_repository_url`, applying the current `normalizeScmUrl`. No POMs are
+ * fetched — the raw SCM value is already in the DB. Fills recoverable NULLs
+ * (Gap B) and clears non-repository values (Gap C) via direct UPDATE.
+ *
+ * Idempotent and resumable: the id cursor is derived from the scan, so a
+ * re-run after an interrupt simply reprocesses from the start and skips rows
+ * that already match.
+ */
+export async function backfillMavenRepositoryUrls(
+  qx: QueryExecutor,
+  options: { batchSize: number; dryRun: boolean; isShuttingDown: () => boolean },
+): Promise<RepoUrlBackfillTotals> {
+  const { batchSize, dryRun, isShuttingDown } = options
+  const totals: RepoUrlBackfillTotals = {
+    scanned: 0,
+    filled: 0,
+    cleared: 0,
+    rewritten: 0,
+    unchanged: 0,
+  }
+
+  let afterId = 0
+  for (;;) {
+    if (isShuttingDown()) {
+      log.info('Shutdown requested — stopping after the current batch.')
+      break
+    }
+
+    const rows = await listMavenPackagesForRepoUrlRecompute(qx, { afterId, limit: batchSize })
+    if (rows.length === 0) break
+
+    const updates: { id: number; repositoryUrl: string | null }[] = []
+    for (const row of rows) {
+      totals.scanned++
+      const desired = normalizeScmUrl(row.declaredRepositoryUrl)
+      if (desired === row.repositoryUrl) {
+        totals.unchanged++
+        continue
+      }
+      if (row.repositoryUrl === null) totals.filled++
+      else if (desired === null) totals.cleared++
+      else totals.rewritten++
+      updates.push({ id: row.id, repositoryUrl: desired })
+    }
+
+    if (updates.length > 0 && !dryRun) {
+      await updateMavenRepositoryUrls(qx, updates)
+    }
+
+    afterId = rows[rows.length - 1].id
+    log.info({ afterId, changes: updates.length, dryRun, ...totals }, 'Backfill progress')
+  }
+
+  return totals
+}

services/apps/packages_worker/src/maven/extract.ts

@@ -453,37 +453,89 @@ export async function extractArtifact(groupId: string, artifactId: string, versi
 // ─── SCM URL normalisation ───────────────────────────────────────────────────
 
 /**
- * Converts the raw SCM URL from a POM (declared_repository_url) into a clean
- * HTTPS repository URL suitable for storage as repository_url.
+ * Known source-code-hosting hosts. A normalised repository_url is only produced
+ * when the URL resolves to one of these — anything else (homepages, doc sites,
+ * placeholders) yields null so it is never stored as a repository link.
+ *
+ * TODO(CM): host list pending product confirmation before rollout.
+ */
+const SCM_HOSTS = new Set([
+  'github.com',
+  'gitlab.com',
+  'bitbucket.org',
+  'gitee.com',
+  'codeberg.org',
+])
+
+/** Hosts whose owner/repo path is case-insensitive and should be lower-cased. */
+const CASE_INSENSITIVE_HOSTS = new Set(['github.com', 'gitlab.com'])
+
+/**
+ * Converts the raw SCM URL from a POM (declared_repository_url) into a clean,
+ * canonical `https://<host>/<owner>/<repo>` repository URL suitable for storage
+ * as repository_url. Returns null when the input does not resolve to a real
+ * repository on a known SCM host.
  *
  * Handles common Maven SCM URL forms:
- *   scm:git:git@github.com:owner/repo.git  → https://github.com/owner/repo
- *   scm:git:https://github.com/owner/repo  → https://github.com/owner/repo
- *   git://github.com/owner/repo.git        → https://github.com/owner/repo
- *   https://github.com/owner/repo/tree/... → https://github.com/owner/repo
+ *   scm:git:git@github.com:owner/repo.git    → https://github.com/owner/repo
+ *   scm:git:https://github.com/owner/repo    → https://github.com/owner/repo
+ *   scm:git:github.com/owner/repo            → https://github.com/owner/repo
+ *   github.com/owner/repo (no scheme)        → https://github.com/owner/repo
+ *   git://github.com/owner/repo.git          → https://github.com/owner/repo
+ *   http://github.com/owner/repo/tree/...    → https://github.com/owner/repo
+ *
+ * Rejected (→ null): website-only URLs (https://meson.ai/), non-SCM hosts
+ * (svn://…, http://source.android.com), placeholders (Private, ${scm-url}).
  */
 export function normalizeScmUrl(raw: string | null): string | null {
   if (!raw) return null
-  let url = raw.trim()
+  let s = raw.trim()
+  if (!s) return null
 
-  // Strip scm:git: or scm: prefix
-  url = url.replace(/^scm:git:/i, '').replace(/^scm:/i, '')
+  // Strip Maven scm:git: / scm: prefix
+  s = s.replace(/^scm:git:/i, '').replace(/^scm:/i, '')
 
-  // Convert SSH git@host:owner/repo → https://host/owner/repo
-  url = url.replace(/^git@([^:]+):(.+)$/, 'https://$1/$2')
+  // git+https://… → https://…
+  s = s.replace(/^git\+/, '')
 
-  // Convert git:// → https://
-  url = url.replace(/^git:\/\//, 'https://')
+  // SCP form git@host:owner/repo → https://host/owner/repo
+  s = s.replace(/^git@([^:/]+):(.+)$/, 'https://$1/$2')
 
-  // Strip trailing .git
-  url = url.replace(/\.git$/, '')
+  // ssh://git@host/… → https://host/…
+  s = s.replace(/^ssh:\/\/git@([^/]+)\//, 'https://$1/')
 
-  // Strip /tree/... or /blob/... path suffixes (keep only host + owner + repo)
-  url = url.replace(/\/(tree|blob)(\/.*)?$/, '')
+  // git:// → https://, and upgrade http:// → https://
+  s = s.replace(/^git:\/\//, 'https://').replace(/^http:\/\//, 'https://')
 
-  if (!url.startsWith('https://')) return null
+  // No scheme at all (e.g. "github.com/owner/repo") → assume https
+  if (!s.includes('://')) s = `https://${s}`
+
+  let parsed: URL
+  try {
+    parsed = new URL(s)
+  } catch {
+    return null
+  }
+
+  if (parsed.protocol !== 'https:') return null
+
+  const host = parsed.hostname.toLowerCase().replace(/^www\./, '')
+  if (!SCM_HOSTS.has(host)) return null
+
+  // Require at least owner + repo path segments
+  const segments = parsed.pathname.split('/').filter(Boolean)
+  if (segments.length < 2) return null
+
+  let owner = segments[0]
+  let name = segments[1].replace(/\.git$/, '')
+  if (!owner || !name) return null
+
+  if (CASE_INSENSITIVE_HOSTS.has(host)) {
+    owner = owner.toLowerCase()
+    name = name.toLowerCase()
+  }
 
-  return url.replace(/\/$/, '')
+  return `https://${host}/${owner}/${name}`
 }
 
 // ─── Private helpers ──────────────────────────────────────────────────────────

services/libs/data-access-layer/src/osspckgs/packages.ts

@@ -118,6 +118,83 @@ export async function listMavenPackagesToSync(
   )
 }
 
+// ─── repository_url backfill ──────────────────────────────────────────────────
+
+export type MavenRepoUrlRow = {
+  id: number
+  declaredRepositoryUrl: string | null
+  repositoryUrl: string | null
+}
+
+/**
+ * Keyset-paginated scan of Maven rows that carry a repository link (declared or
+ * canonical). Rows with neither are skipped — there is nothing to recompute.
+ * Used by the repository_url backfill to re-run the normalizer over stored data
+ * without re-fetching POMs from the registry.
+ */
+export async function listMavenPackagesForRepoUrlRecompute(
+  qx: QueryExecutor,
+  options: { afterId: number; limit: number },
+): Promise<MavenRepoUrlRow[]> {
+  return qx.select(
+    `
+    SELECT
+      id,
+      declared_repository_url AS "declaredRepositoryUrl",
+      repository_url          AS "repositoryUrl"
+    FROM packages
+    WHERE ecosystem = 'maven'
+      AND id > $(afterId)
+      AND (declared_repository_url IS NOT NULL OR repository_url IS NOT NULL)
+    ORDER BY id ASC
+    LIMIT $(limit)
+    `,
+    { afterId: options.afterId, limit: options.limit },
+  )
+}
+
+/**
+ * Applies a batch of recomputed repository_url values via direct UPDATE — the
+ * only way to clear a stale value, since the enrichment upsert COALESCEs and
+ * cannot write NULL. Splits clears (→ NULL) from sets to avoid NULLs inside a
+ * text[] array literal.
+ */
+export async function updateMavenRepositoryUrls(
+  qx: QueryExecutor,
+  updates: { id: number; repositoryUrl: string | null }[],
+): Promise<void> {
+  if (updates.length === 0) return
+
+  const toClear = updates.filter((u) => u.repositoryUrl === null).map((u) => u.id)
+  const toSet = updates.filter(
+    (u): u is { id: number; repositoryUrl: string } => u.repositoryUrl !== null,
+  )
+
+  if (toClear.length > 0) {
+    await qx.result(
+      `UPDATE packages SET repository_url = NULL
+       WHERE id = ANY($(ids)::bigint[]) AND repository_url IS NOT NULL`,
+      { ids: toClear },
+    )
+  }
+
+  if (toSet.length > 0) {
+    await qx.result(
+      `
+      UPDATE packages p
+      SET repository_url = v.repository_url
+      FROM (
+        SELECT unnest($(ids)::bigint[]) AS id,
+               unnest($(urls)::text[])  AS repository_url
+      ) v
+      WHERE p.id = v.id
+        AND p.repository_url IS DISTINCT FROM v.repository_url
+      `,
+      { ids: toSet.map((u) => u.id), urls: toSet.map((u) => u.repositoryUrl) },
+    )
+  }
+}
+
 // ─── packages touch ───────────────────────────────────────────────────────────
 
 /**