Merge branch 'main' into cm-361-affiliation-registry-migration

Author: skwowet

services/apps/packages_worker/src/maven/__tests__/registry.test.ts

@@ -1,7 +1,11 @@
 import { describe, expect, it } from 'vitest'
 
 import {
+  GRADLE_PLUGIN_PORTAL_BASE_URL,
+  JITPACK_BASE_URL,
   MAVEN_CENTRAL_BASE_URL,
+  isJitpackNamespace,
+  jitpackPageUrl,
   resolveRegistryBaseUrl,
   resolveRegistryPageUrl,
   resolveRegistryPageUrlFromBase,
@@ -148,6 +152,22 @@ describe('resolveRegistryBaseUrl', () => {
       'https://repo.jenkins-ci.org/public',
     )
   })
+
+  it('returns Maven Central for io.github namespace (JitPack is a fallback, not primary)', () => {
+    expect(resolveRegistryBaseUrl('io.github.resilience4j')).toBe(MAVEN_CENTRAL_BASE_URL)
+  })
+
+  it('returns Maven Central for bare io.github', () => {
+    expect(resolveRegistryBaseUrl('io.github')).toBe(MAVEN_CENTRAL_BASE_URL)
+  })
+
+  it('returns Maven Central for com.github namespace (JitPack is a fallback, not primary)', () => {
+    expect(resolveRegistryBaseUrl('com.github.ben-manes')).toBe(MAVEN_CENTRAL_BASE_URL)
+  })
+
+  it('returns Maven Central for io.githubfoo (no dot boundary)', () => {
+    expect(resolveRegistryBaseUrl('io.githubfoo')).toBe(MAVEN_CENTRAL_BASE_URL)
+  })
 })
 
 describe('resolveRegistryPageUrl', () => {
@@ -174,6 +194,18 @@ describe('resolveRegistryPageUrl', () => {
       'https://central.sonatype.com/artifact/org.apache.commons/commons-lang3',
     )
   })
+
+  it('returns Maven Central URL for io.github (Central is primary)', () => {
+    expect(resolveRegistryPageUrl('io.github.resilience4j', 'resilience4j-core')).toBe(
+      'https://central.sonatype.com/artifact/io.github.resilience4j/resilience4j-core',
+    )
+  })
+
+  it('returns Maven Central URL for com.github (Central is primary)', () => {
+    expect(resolveRegistryPageUrl('com.github.ben-manes', 'caffeine')).toBe(
+      'https://central.sonatype.com/artifact/com.github.ben-manes/caffeine',
+    )
+  })
 })
 
 describe('resolveRegistryPageUrlFromBase', () => {
@@ -193,4 +225,65 @@ describe('resolveRegistryPageUrlFromBase', () => {
       resolveRegistryPageUrlFromBase('com.google.firebase', 'firebase-analytics', googleMavenUrl),
     ).toBe('https://maven.google.com/web/index.html#com.google.firebase:firebase-analytics')
   })
+
+  it('returns Gradle Plugin Portal path when resolvedBaseUrl is Gradle Plugin Portal', () => {
+    expect(
+      resolveRegistryPageUrlFromBase(
+        'io.github.trueangle',
+        'gradle-plugin',
+        GRADLE_PLUGIN_PORTAL_BASE_URL,
+      ),
+    ).toBe('https://plugins.gradle.org/m2/io/github/trueangle/gradle-plugin/')
+  })
+
+  it('returns JitPack browse URL when resolvedBaseUrl is JitPack', () => {
+    expect(
+      resolveRegistryPageUrlFromBase('io.github.trueangle', 'gradle-plugin', JITPACK_BASE_URL),
+    ).toBe('https://jitpack.io/#trueangle/gradle-plugin')
+  })
+})
+
+describe('isJitpackNamespace', () => {
+  it('returns true for io.github.* packages', () => {
+    expect(isJitpackNamespace('io.github.resilience4j')).toBe(true)
+  })
+
+  it('returns true for bare io.github', () => {
+    expect(isJitpackNamespace('io.github')).toBe(true)
+  })
+
+  it('returns true for com.github.* packages', () => {
+    expect(isJitpackNamespace('com.github.ben-manes')).toBe(true)
+  })
+
+  it('returns true for bare com.github', () => {
+    expect(isJitpackNamespace('com.github')).toBe(true)
+  })
+
+  it('returns false for io.githubfoo (no dot boundary)', () => {
+    expect(isJitpackNamespace('io.githubfoo')).toBe(false)
+  })
+
+  it('returns false for unrelated namespaces', () => {
+    expect(isJitpackNamespace('org.apache.commons')).toBe(false)
+    expect(isJitpackNamespace('io.jenkins.plugins')).toBe(false)
+  })
+})
+
+describe('jitpackPageUrl', () => {
+  it('strips io.github. prefix from groupId', () => {
+    expect(jitpackPageUrl('io.github.resilience4j', 'resilience4j-core')).toBe(
+      'https://jitpack.io/#resilience4j/resilience4j-core',
+    )
+  })
+
+  it('strips com.github. prefix from groupId', () => {
+    expect(jitpackPageUrl('com.github.ben-manes', 'caffeine')).toBe(
+      'https://jitpack.io/#ben-manes/caffeine',
+    )
+  })
+
+  it('handles bare io.github with no sub-namespace', () => {
+    expect(jitpackPageUrl('io.github', 'some-lib')).toBe('https://jitpack.io/#/some-lib')
+  })
 })

services/apps/packages_worker/src/maven/metadata.ts

@@ -12,7 +12,7 @@ import axios from 'axios'
 import { XMLParser } from 'fast-xml-parser'
 
 import { isPrerelease } from './normalize'
-import { resolveRegistryBaseUrl } from './registry'
+import { JITPACK_BASE_URL, resolveRegistryBaseUrl } from './registry'
 
 const REQUEST_TIMEOUT_MS = 10_000
 const MAX_RETRIES = 3
@@ -111,7 +111,12 @@ export async function resolveVersionsList(
     } catch (err) {
       if (axios.isAxiosError(err)) {
         const status = err.response?.status
-        if (status === 404) return { kind: 'NOT_FOUND' }
+        // 404 = standard not-found. 401 = JitPack-specific: returned for packages
+        // that don't exist as public builds. Scoped to JitPack only — other
+        // registries may return 401 for auth failures, which should not be
+        // silently treated as a missing package.
+        if (status === 404 || (status === 401 && url.startsWith(JITPACK_BASE_URL + '/')))
+          return { kind: 'NOT_FOUND' }
         // 429 = explicit rate limit, 403 = CDN throttle (Maven Central uses both)
         if ((status === 429 || status === 403) && attempt < MAX_RETRIES) {
           const delay = RETRY_BASE_MS * 2 ** attempt + Math.random() * 500

services/apps/packages_worker/src/maven/registry.ts

@@ -9,9 +9,17 @@
  *
  * All these repos expose the standard Maven repository layout, so the same
  * metadata.xml / POM fetch logic works — only the base URL changes.
+ *
+ * JitPack (io.github.*, com.github.*) is NOT listed here as a primary registry
+ * because many well-known artifacts under those prefixes are published on Maven
+ * Central (e.g. io.github.resilience4j, com.github.ben-manes:caffeine). Maven
+ * Central is tried first; JitPack is used only as a fallback when Central returns
+ * NOT_FOUND (see the enrichment loop).
  */
 
 export const MAVEN_CENTRAL_BASE_URL = 'https://repo1.maven.org/maven2'
+export const GRADLE_PLUGIN_PORTAL_BASE_URL = 'https://plugins.gradle.org/m2'
+export const JITPACK_BASE_URL = 'https://jitpack.io'
 
 interface RegistryEntry {
   prefix: string
@@ -96,8 +104,8 @@ const ALTERNATIVE_REGISTRIES: RegistryEntry[] = [
   },
   {
     prefix: 'gradle.plugin',
-    baseUrl: 'https://plugins.gradle.org/m2',
-    pageUrl: (g, a) => `https://plugins.gradle.org/m2/${g.replace(/\./g, '/')}/${a}/`,
+    baseUrl: GRADLE_PLUGIN_PORTAL_BASE_URL,
+    pageUrl: (g, a) => `${GRADLE_PLUGIN_PORTAL_BASE_URL}/${g.replace(/\./g, '/')}/${a}/`,
   },
   {
     prefix: 'com.cloudbees.plugins',
@@ -166,6 +174,30 @@ export function isAlternativeRegistry(groupId: string): boolean {
   return findEntry(groupId) !== undefined
 }
 
+/**
+ * Returns true for io.github.* and com.github.* groupIds — the JitPack namespace
+ * convention. Maven Central is tried first for these; JitPack is the fallback.
+ */
+export function isJitpackNamespace(groupId: string): boolean {
+  return (
+    groupId === 'io.github' ||
+    groupId.startsWith('io.github.') ||
+    groupId === 'com.github' ||
+    groupId.startsWith('com.github.')
+  )
+}
+
+/** Returns a human-browsable JitPack URL for an io.github.* or com.github.* artifact. */
+export function jitpackPageUrl(groupId: string, artifactId: string): string {
+  if (groupId.startsWith('io.github.')) {
+    return `${JITPACK_BASE_URL}/#${groupId.slice('io.github.'.length)}/${artifactId}`
+  }
+  if (groupId.startsWith('com.github.')) {
+    return `${JITPACK_BASE_URL}/#${groupId.slice('com.github.'.length)}/${artifactId}`
+  }
+  return `${JITPACK_BASE_URL}/#/${artifactId}`
+}
+
 /**
  * Returns a human-browsable URL for the given artifact in its authoritative registry.
  */
@@ -192,5 +224,11 @@ export function resolveRegistryPageUrlFromBase(
   if (isCentral) {
     return `https://central.sonatype.com/artifact/${groupId}/${artifactId}`
   }
+  if (resolvedBaseUrl === GRADLE_PLUGIN_PORTAL_BASE_URL) {
+    return `${GRADLE_PLUGIN_PORTAL_BASE_URL}/${groupId.replace(/\./g, '/')}/${artifactId}/`
+  }
+  if (resolvedBaseUrl === JITPACK_BASE_URL) {
+    return jitpackPageUrl(groupId, artifactId)
+  }
   return resolveRegistryPageUrl(groupId, artifactId)
 }

services/apps/packages_worker/src/maven/runMavenEnrichmentLoop.ts

@@ -19,8 +19,11 @@ import { extractArtifact, getPomCacheStats, normalizeScmUrl } from './extract'
 import { isMavenFetchError, resolveVersionsList } from './metadata'
 import { isPrerelease, parseRepoUrl } from './normalize'
 import {
+  GRADLE_PLUGIN_PORTAL_BASE_URL,
+  JITPACK_BASE_URL,
   MAVEN_CENTRAL_BASE_URL,
   isAlternativeRegistry,
+  isJitpackNamespace,
   resolveRegistryBaseUrl,
   resolveRegistryPageUrl,
   resolveRegistryPageUrlFromBase,
@@ -141,6 +144,32 @@ async function processCriticalPackage(qx: QueryExecutor, pkg: PackageRow, forceF
     }
   }
 
+  // Second fallback: JitPack for io.github.* / com.github.* packages not found on
+  // Maven Central. These namespaces are also used by well-known Central artifacts
+  // (e.g. resilience4j, caffeine), so Central is tried first; JitPack is the fallback
+  // for packages that genuinely live only on JitPack.
+  if (isMavenFetchError(metadata) && metadata.kind === 'NOT_FOUND' && isJitpackNamespace(groupId)) {
+    const jitpackMetadata = await resolveVersionsList(groupId, artifactId, JITPACK_BASE_URL)
+    if (!isMavenFetchError(jitpackMetadata)) {
+      log.info({ groupId, artifactId }, 'Not found on Maven Central — resolved via JitPack fallback')
+      baseUrl = JITPACK_BASE_URL
+      metadata = jitpackMetadata
+    }
+  }
+
+  // Third fallback: Gradle Plugin Portal. Plugins can be published under any groupId
+  // (not just gradle.plugin.*), so this is worth trying for all NOT_FOUND packages.
+  // Skip when GPP is already the primary registry (gradle.plugin.*) to avoid a
+  // redundant second call to the same URL.
+  if (isMavenFetchError(metadata) && metadata.kind === 'NOT_FOUND' && baseUrl !== GRADLE_PLUGIN_PORTAL_BASE_URL) {
+    const gradlePortalMetadata = await resolveVersionsList(groupId, artifactId, GRADLE_PLUGIN_PORTAL_BASE_URL)
+    if (!isMavenFetchError(gradlePortalMetadata)) {
+      log.info({ groupId, artifactId }, 'Not found in primary/Central — resolved via Gradle Plugin Portal fallback')
+      baseUrl = GRADLE_PLUGIN_PORTAL_BASE_URL
+      metadata = gradlePortalMetadata
+    }
+  }
+
   if (isMavenFetchError(metadata)) {
     if (metadata.kind === 'NOT_FOUND') {
       await upsertPackage(qx, {