diff --git a/services/apps/packages_worker/src/maven/__tests__/registry.test.ts b/services/apps/packages_worker/src/maven/__tests__/registry.test.ts index e2aeee6235..11919ad086 100644 --- a/services/apps/packages_worker/src/maven/__tests__/registry.test.ts +++ b/services/apps/packages_worker/src/maven/__tests__/registry.test.ts @@ -1,7 +1,11 @@ import { describe, expect, it } from 'vitest' import { + GRADLE_PLUGIN_PORTAL_BASE_URL, + JITPACK_BASE_URL, MAVEN_CENTRAL_BASE_URL, + isJitpackNamespace, + jitpackPageUrl, resolveRegistryBaseUrl, resolveRegistryPageUrl, resolveRegistryPageUrlFromBase, @@ -148,6 +152,22 @@ describe('resolveRegistryBaseUrl', () => { 'https://repo.jenkins-ci.org/public', ) }) + + it('returns Maven Central for io.github namespace (JitPack is a fallback, not primary)', () => { + expect(resolveRegistryBaseUrl('io.github.resilience4j')).toBe(MAVEN_CENTRAL_BASE_URL) + }) + + it('returns Maven Central for bare io.github', () => { + expect(resolveRegistryBaseUrl('io.github')).toBe(MAVEN_CENTRAL_BASE_URL) + }) + + it('returns Maven Central for com.github namespace (JitPack is a fallback, not primary)', () => { + expect(resolveRegistryBaseUrl('com.github.ben-manes')).toBe(MAVEN_CENTRAL_BASE_URL) + }) + + it('returns Maven Central for io.githubfoo (no dot boundary)', () => { + expect(resolveRegistryBaseUrl('io.githubfoo')).toBe(MAVEN_CENTRAL_BASE_URL) + }) }) describe('resolveRegistryPageUrl', () => { @@ -174,6 +194,18 @@ describe('resolveRegistryPageUrl', () => { 'https://central.sonatype.com/artifact/org.apache.commons/commons-lang3', ) }) + + it('returns Maven Central URL for io.github (Central is primary)', () => { + expect(resolveRegistryPageUrl('io.github.resilience4j', 'resilience4j-core')).toBe( + 'https://central.sonatype.com/artifact/io.github.resilience4j/resilience4j-core', + ) + }) + + it('returns Maven Central URL for com.github (Central is primary)', () => { + expect(resolveRegistryPageUrl('com.github.ben-manes', 'caffeine')).toBe( + 'https://central.sonatype.com/artifact/com.github.ben-manes/caffeine', + ) + }) }) describe('resolveRegistryPageUrlFromBase', () => { @@ -193,4 +225,65 @@ describe('resolveRegistryPageUrlFromBase', () => { resolveRegistryPageUrlFromBase('com.google.firebase', 'firebase-analytics', googleMavenUrl), ).toBe('https://maven.google.com/web/index.html#com.google.firebase:firebase-analytics') }) + + it('returns Gradle Plugin Portal path when resolvedBaseUrl is Gradle Plugin Portal', () => { + expect( + resolveRegistryPageUrlFromBase( + 'io.github.trueangle', + 'gradle-plugin', + GRADLE_PLUGIN_PORTAL_BASE_URL, + ), + ).toBe('https://plugins.gradle.org/m2/io/github/trueangle/gradle-plugin/') + }) + + it('returns JitPack browse URL when resolvedBaseUrl is JitPack', () => { + expect( + resolveRegistryPageUrlFromBase('io.github.trueangle', 'gradle-plugin', JITPACK_BASE_URL), + ).toBe('https://jitpack.io/#trueangle/gradle-plugin') + }) +}) + +describe('isJitpackNamespace', () => { + it('returns true for io.github.* packages', () => { + expect(isJitpackNamespace('io.github.resilience4j')).toBe(true) + }) + + it('returns true for bare io.github', () => { + expect(isJitpackNamespace('io.github')).toBe(true) + }) + + it('returns true for com.github.* packages', () => { + expect(isJitpackNamespace('com.github.ben-manes')).toBe(true) + }) + + it('returns true for bare com.github', () => { + expect(isJitpackNamespace('com.github')).toBe(true) + }) + + it('returns false for io.githubfoo (no dot boundary)', () => { + expect(isJitpackNamespace('io.githubfoo')).toBe(false) + }) + + it('returns false for unrelated namespaces', () => { + expect(isJitpackNamespace('org.apache.commons')).toBe(false) + expect(isJitpackNamespace('io.jenkins.plugins')).toBe(false) + }) +}) + +describe('jitpackPageUrl', () => { + it('strips io.github. prefix from groupId', () => { + expect(jitpackPageUrl('io.github.resilience4j', 'resilience4j-core')).toBe( + 'https://jitpack.io/#resilience4j/resilience4j-core', + ) + }) + + it('strips com.github. prefix from groupId', () => { + expect(jitpackPageUrl('com.github.ben-manes', 'caffeine')).toBe( + 'https://jitpack.io/#ben-manes/caffeine', + ) + }) + + it('handles bare io.github with no sub-namespace', () => { + expect(jitpackPageUrl('io.github', 'some-lib')).toBe('https://jitpack.io/#/some-lib') + }) }) diff --git a/services/apps/packages_worker/src/maven/metadata.ts b/services/apps/packages_worker/src/maven/metadata.ts index 6833abe370..3c10445285 100644 --- a/services/apps/packages_worker/src/maven/metadata.ts +++ b/services/apps/packages_worker/src/maven/metadata.ts @@ -12,7 +12,7 @@ import axios from 'axios' import { XMLParser } from 'fast-xml-parser' import { isPrerelease } from './normalize' -import { resolveRegistryBaseUrl } from './registry' +import { JITPACK_BASE_URL, resolveRegistryBaseUrl } from './registry' const REQUEST_TIMEOUT_MS = 10_000 const MAX_RETRIES = 3 @@ -111,7 +111,12 @@ export async function resolveVersionsList( } catch (err) { if (axios.isAxiosError(err)) { const status = err.response?.status - if (status === 404) return { kind: 'NOT_FOUND' } + // 404 = standard not-found. 401 = JitPack-specific: returned for packages + // that don't exist as public builds. Scoped to JitPack only — other + // registries may return 401 for auth failures, which should not be + // silently treated as a missing package. + if (status === 404 || (status === 401 && url.startsWith(JITPACK_BASE_URL + '/'))) + return { kind: 'NOT_FOUND' } // 429 = explicit rate limit, 403 = CDN throttle (Maven Central uses both) if ((status === 429 || status === 403) && attempt < MAX_RETRIES) { const delay = RETRY_BASE_MS * 2 ** attempt + Math.random() * 500 diff --git a/services/apps/packages_worker/src/maven/registry.ts b/services/apps/packages_worker/src/maven/registry.ts index fb066f6deb..507282b799 100644 --- a/services/apps/packages_worker/src/maven/registry.ts +++ b/services/apps/packages_worker/src/maven/registry.ts @@ -9,9 +9,17 @@ * * All these repos expose the standard Maven repository layout, so the same * metadata.xml / POM fetch logic works — only the base URL changes. + * + * JitPack (io.github.*, com.github.*) is NOT listed here as a primary registry + * because many well-known artifacts under those prefixes are published on Maven + * Central (e.g. io.github.resilience4j, com.github.ben-manes:caffeine). Maven + * Central is tried first; JitPack is used only as a fallback when Central returns + * NOT_FOUND (see the enrichment loop). */ export const MAVEN_CENTRAL_BASE_URL = 'https://repo1.maven.org/maven2' +export const GRADLE_PLUGIN_PORTAL_BASE_URL = 'https://plugins.gradle.org/m2' +export const JITPACK_BASE_URL = 'https://jitpack.io' interface RegistryEntry { prefix: string @@ -96,8 +104,8 @@ const ALTERNATIVE_REGISTRIES: RegistryEntry[] = [ }, { prefix: 'gradle.plugin', - baseUrl: 'https://plugins.gradle.org/m2', - pageUrl: (g, a) => `https://plugins.gradle.org/m2/${g.replace(/\./g, '/')}/${a}/`, + baseUrl: GRADLE_PLUGIN_PORTAL_BASE_URL, + pageUrl: (g, a) => `${GRADLE_PLUGIN_PORTAL_BASE_URL}/${g.replace(/\./g, '/')}/${a}/`, }, { prefix: 'com.cloudbees.plugins', @@ -166,6 +174,30 @@ export function isAlternativeRegistry(groupId: string): boolean { return findEntry(groupId) !== undefined } +/** + * Returns true for io.github.* and com.github.* groupIds — the JitPack namespace + * convention. Maven Central is tried first for these; JitPack is the fallback. + */ +export function isJitpackNamespace(groupId: string): boolean { + return ( + groupId === 'io.github' || + groupId.startsWith('io.github.') || + groupId === 'com.github' || + groupId.startsWith('com.github.') + ) +} + +/** Returns a human-browsable JitPack URL for an io.github.* or com.github.* artifact. */ +export function jitpackPageUrl(groupId: string, artifactId: string): string { + if (groupId.startsWith('io.github.')) { + return `${JITPACK_BASE_URL}/#${groupId.slice('io.github.'.length)}/${artifactId}` + } + if (groupId.startsWith('com.github.')) { + return `${JITPACK_BASE_URL}/#${groupId.slice('com.github.'.length)}/${artifactId}` + } + return `${JITPACK_BASE_URL}/#/${artifactId}` +} + /** * Returns a human-browsable URL for the given artifact in its authoritative registry. */ @@ -192,5 +224,11 @@ export function resolveRegistryPageUrlFromBase( if (isCentral) { return `https://central.sonatype.com/artifact/${groupId}/${artifactId}` } + if (resolvedBaseUrl === GRADLE_PLUGIN_PORTAL_BASE_URL) { + return `${GRADLE_PLUGIN_PORTAL_BASE_URL}/${groupId.replace(/\./g, '/')}/${artifactId}/` + } + if (resolvedBaseUrl === JITPACK_BASE_URL) { + return jitpackPageUrl(groupId, artifactId) + } return resolveRegistryPageUrl(groupId, artifactId) } diff --git a/services/apps/packages_worker/src/maven/runMavenEnrichmentLoop.ts b/services/apps/packages_worker/src/maven/runMavenEnrichmentLoop.ts index 635b4c0580..7efdbd97e1 100644 --- a/services/apps/packages_worker/src/maven/runMavenEnrichmentLoop.ts +++ b/services/apps/packages_worker/src/maven/runMavenEnrichmentLoop.ts @@ -19,8 +19,11 @@ import { extractArtifact, getPomCacheStats, normalizeScmUrl } from './extract' import { isMavenFetchError, resolveVersionsList } from './metadata' import { isPrerelease, parseRepoUrl } from './normalize' import { + GRADLE_PLUGIN_PORTAL_BASE_URL, + JITPACK_BASE_URL, MAVEN_CENTRAL_BASE_URL, isAlternativeRegistry, + isJitpackNamespace, resolveRegistryBaseUrl, resolveRegistryPageUrl, resolveRegistryPageUrlFromBase, @@ -141,6 +144,32 @@ async function processCriticalPackage(qx: QueryExecutor, pkg: PackageRow, forceF } } + // Second fallback: JitPack for io.github.* / com.github.* packages not found on + // Maven Central. These namespaces are also used by well-known Central artifacts + // (e.g. resilience4j, caffeine), so Central is tried first; JitPack is the fallback + // for packages that genuinely live only on JitPack. + if (isMavenFetchError(metadata) && metadata.kind === 'NOT_FOUND' && isJitpackNamespace(groupId)) { + const jitpackMetadata = await resolveVersionsList(groupId, artifactId, JITPACK_BASE_URL) + if (!isMavenFetchError(jitpackMetadata)) { + log.info({ groupId, artifactId }, 'Not found on Maven Central — resolved via JitPack fallback') + baseUrl = JITPACK_BASE_URL + metadata = jitpackMetadata + } + } + + // Third fallback: Gradle Plugin Portal. Plugins can be published under any groupId + // (not just gradle.plugin.*), so this is worth trying for all NOT_FOUND packages. + // Skip when GPP is already the primary registry (gradle.plugin.*) to avoid a + // redundant second call to the same URL. + if (isMavenFetchError(metadata) && metadata.kind === 'NOT_FOUND' && baseUrl !== GRADLE_PLUGIN_PORTAL_BASE_URL) { + const gradlePortalMetadata = await resolveVersionsList(groupId, artifactId, GRADLE_PLUGIN_PORTAL_BASE_URL) + if (!isMavenFetchError(gradlePortalMetadata)) { + log.info({ groupId, artifactId }, 'Not found in primary/Central — resolved via Gradle Plugin Portal fallback') + baseUrl = GRADLE_PLUGIN_PORTAL_BASE_URL + metadata = gradlePortalMetadata + } + } + if (isMavenFetchError(metadata)) { if (metadata.kind === 'NOT_FOUND') { await upsertPackage(qx, {