Skip to content

Commit 962b277

Browse files
ulemonsskwowet
ulemons
authored and
skwowet
committed
fix: add missing registries (CM-1299) (#4274)
Signed-off-by: Umberto Sgueglia <usgueglia@contractor.linuxfoundation.org> Signed-off-by: Yeganathan S <63534555+skwowet@users.noreply.github.com>
1 parent 3134b8d commit 962b277

5 files changed

Lines changed: 403 additions & 98 deletions

File tree

‎services/apps/packages_worker/src/maven/__tests__/registry.test.ts‎

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
import { describe, expect, it } from 'vitest'
2+
3+
import { resolveRegistryBaseUrl, resolveRegistryPageUrl } from '../registry'
4+
5+
describe('resolveRegistryBaseUrl', () => {
6+
it('returns Google Maven for androidx namespace', () => {
7+
expect(resolveRegistryBaseUrl('androidx.annotation')).toBe(
8+
'https://dl.google.com/dl/android/maven2',
9+
)
10+
})
11+
12+
it('returns Google Maven for com.google.android.gms sub-namespace', () => {
13+
expect(resolveRegistryBaseUrl('com.google.android.gms')).toBe(
14+
'https://dl.google.com/dl/android/maven2',
15+
)
16+
})
17+
18+
it('returns Maven Central for bare com.google.android (legacy SDK stubs on Central)', () => {
19+
expect(resolveRegistryBaseUrl('com.google.android')).toBe('https://repo1.maven.org/maven2')
20+
})
21+
22+
it('returns Google Maven for android.arch (pre-AndroidX Architecture Components)', () => {
23+
expect(resolveRegistryBaseUrl('android.arch.core')).toBe(
24+
'https://dl.google.com/dl/android/maven2',
25+
)
26+
})
27+
28+
it('returns Google Maven for com.android.support (pre-AndroidX support library)', () => {
29+
expect(resolveRegistryBaseUrl('com.android.support')).toBe(
30+
'https://dl.google.com/dl/android/maven2',
31+
)
32+
})
33+
34+
it('returns Google Maven for com.google.testing.platform', () => {
35+
expect(resolveRegistryBaseUrl('com.google.testing.platform')).toBe(
36+
'https://dl.google.com/dl/android/maven2',
37+
)
38+
})
39+
40+
it('returns Google Maven for com.google.firebase namespace', () => {
41+
expect(resolveRegistryBaseUrl('com.google.firebase')).toBe(
42+
'https://dl.google.com/dl/android/maven2',
43+
)
44+
})
45+
46+
it('returns Google Maven for com.google.mlkit namespace', () => {
47+
expect(resolveRegistryBaseUrl('com.google.mlkit')).toBe(
48+
'https://dl.google.com/dl/android/maven2',
49+
)
50+
})
51+
52+
it('returns Gradle Plugin Portal for gradle.plugin namespace', () => {
53+
expect(resolveRegistryBaseUrl('gradle.plugin.name.remal')).toBe('https://plugins.gradle.org/m2')
54+
})
55+
56+
it('returns Jenkins repo for org.kohsuke.stapler (Jenkins HTTP framework)', () => {
57+
expect(resolveRegistryBaseUrl('org.kohsuke.stapler')).toBe('https://repo.jenkins-ci.org/public')
58+
})
59+
60+
it('returns Jenkins repo for org.jenkins-ci namespace', () => {
61+
expect(resolveRegistryBaseUrl('org.jenkins-ci.main')).toBe('https://repo.jenkins-ci.org/public')
62+
})
63+
64+
it('returns Jenkins repo for io.jenkins.plugins namespace', () => {
65+
expect(resolveRegistryBaseUrl('io.jenkins.plugins')).toBe('https://repo.jenkins-ci.org/public')
66+
})
67+
68+
it('returns Jenkins repo for io.jenkins.blueocean namespace', () => {
69+
expect(resolveRegistryBaseUrl('io.jenkins.blueocean')).toBe(
70+
'https://repo.jenkins-ci.org/public',
71+
)
72+
})
73+
74+
it('returns Maven Central for io.jenkins.tools (publishes on Central, not Jenkins repo)', () => {
75+
expect(resolveRegistryBaseUrl('io.jenkins.tools')).toBe('https://repo1.maven.org/maven2')
76+
})
77+
78+
it('returns Maven Central for unknown namespace', () => {
79+
expect(resolveRegistryBaseUrl('org.apache.commons')).toBe('https://repo1.maven.org/maven2')
80+
})
81+
82+
it('returns Google Maven for com.android.tools.analytics-library (Android Studio analytics)', () => {
83+
expect(resolveRegistryBaseUrl('com.android.tools.analytics-library')).toBe(
84+
'https://dl.google.com/dl/android/maven2',
85+
)
86+
})
87+
88+
it('returns Google Maven for com.android.tools.adblib (Android Debug Bridge Library)', () => {
89+
expect(resolveRegistryBaseUrl('com.android.tools.adblib')).toBe(
90+
'https://dl.google.com/dl/android/maven2',
91+
)
92+
})
93+
94+
it('returns Google Maven for com.android.tools.build (Android Gradle Plugin)', () => {
95+
expect(resolveRegistryBaseUrl('com.android.tools.build')).toBe(
96+
'https://dl.google.com/dl/android/maven2',
97+
)
98+
})
99+
100+
it('returns Google Maven for com.android.tools.build.jetifier (AndroidX migration tool)', () => {
101+
expect(resolveRegistryBaseUrl('com.android.tools.build.jetifier')).toBe(
102+
'https://dl.google.com/dl/android/maven2',
103+
)
104+
})
105+
106+
it('returns Google Maven for com.android.tools.utp (Android Unified Test Platform)', () => {
107+
expect(resolveRegistryBaseUrl('com.android.tools.utp')).toBe(
108+
'https://dl.google.com/dl/android/maven2',
109+
)
110+
})
111+
112+
it('returns Maven Central for bare com.android.tools (ddmlib, lint-api publish on Central)', () => {
113+
expect(resolveRegistryBaseUrl('com.android.tools')).toBe('https://repo1.maven.org/maven2')
114+
})
115+
116+
it('returns Google Maven for com.android.identity', () => {
117+
expect(resolveRegistryBaseUrl('com.android.identity')).toBe(
118+
'https://dl.google.com/dl/android/maven2',
119+
)
120+
})
121+
122+
it('returns Google Maven for com.google.mediapipe', () => {
123+
expect(resolveRegistryBaseUrl('com.google.mediapipe')).toBe(
124+
'https://dl.google.com/dl/android/maven2',
125+
)
126+
})
127+
128+
it('returns Jenkins repo for org.jenkinsci.plugins', () => {
129+
expect(resolveRegistryBaseUrl('org.jenkinsci.plugins')).toBe(
130+
'https://repo.jenkins-ci.org/public',
131+
)
132+
})
133+
134+
it('returns Jenkins repo for com.cloudbees.plugins', () => {
135+
expect(resolveRegistryBaseUrl('com.cloudbees.plugins')).toBe(
136+
'https://repo.jenkins-ci.org/public',
137+
)
138+
})
139+
})
140+
141+
describe('resolveRegistryPageUrl', () => {
142+
it('returns Google Maven browse URL for androidx', () => {
143+
expect(resolveRegistryPageUrl('androidx.annotation', 'annotation')).toBe(
144+
'https://maven.google.com/web/index.html#androidx.annotation:annotation',
145+
)
146+
})
147+
148+
it('returns Gradle Plugin Portal URL for gradle.plugin', () => {
149+
expect(resolveRegistryPageUrl('gradle.plugin.name.remal', 'gradle-plugins')).toBe(
150+
'https://plugins.gradle.org/m2/gradle/plugin/name/remal/gradle-plugins/',
151+
)
152+
})
153+
154+
it('returns Jenkins browse URL for org.jenkins-ci', () => {
155+
expect(resolveRegistryPageUrl('org.jenkins-ci.main', 'jenkins-core')).toBe(
156+
'https://repo.jenkins-ci.org/public/org/jenkins-ci/main/jenkins-core/',
157+
)
158+
})
159+
160+
it('returns Maven Central URL for unknown namespace', () => {
161+
expect(resolveRegistryPageUrl('org.apache.commons', 'commons-lang3')).toBe(
162+
'https://central.sonatype.com/artifact/org.apache.commons/commons-lang3',
163+
)
164+
})
165+
})

‎services/apps/packages_worker/src/maven/extract.ts‎

Lines changed: 43 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ import { XMLParser } from 'fast-xml-parser'
77

88
import { getServiceChildLogger } from '@crowd/logging'
99

10+
import { resolveRegistryBaseUrl } from './registry'
11+
1012
const log = getServiceChildLogger('maven')
1113

1214
// ─── Types ────────────────────────────────────────────────────────────────────
@@ -56,14 +58,6 @@ interface PomPerson {
5658

5759
// ─── Config ───────────────────────────────────────────────────────────────────
5860

59-
// Lazy getter — evaluated at call time so the entry point can set MAVEN_FETCHER_BASE_URL
60-
// before the first HTTP request without worrying about module load order.
61-
// Backfill sets MAVEN_FETCHER_BASE_URL from MAVEN_FETCHER_BASE_URL_BACKFILL (GCS mirror);
62-
// the incremental Temporal worker sets it from MAVEN_FETCHER_BASE_URL_INCREMENTAL (repo1).
63-
function getMavenRepo(): string {
64-
return process.env.MAVEN_FETCHER_BASE_URL ?? 'https://repo1.maven.org/maven2'
65-
}
66-
6761
const REQUEST_TIMEOUT_MS = 15_000
6862

6963
const parser = new XMLParser({
@@ -78,10 +72,12 @@ const parser = new XMLParser({
7872
const MAX_RETRIES = 3
7973
const RETRY_BASE_MS = 2_000
8074

75+
// prettier-ignore
8176
async function sleep(ms: number): Promise<void> {
8277
return new Promise((r) => setTimeout(r, ms))
8378
}
8479

80+
// prettier-ignore
8581
async function getWithRetry(url: string): Promise<string> {
8682
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
8783
try {
@@ -108,17 +104,18 @@ async function getWithRetry(url: string): Promise<string> {
108104

109105
// ─── POM fetch ────────────────────────────────────────────────────────────────
110106

111-
export function buildPomUrl(groupId: string, artifactId: string, version: string): string {
112-
const groupPath = groupId.replace(/\./g, '/')
113-
return `${getMavenRepo()}/${groupPath}/${artifactId}/${version}/${artifactId}-${version}.pom`
114-
}
115-
116-
export async function fetchPom(
107+
export function buildPomUrl(
117108
groupId: string,
118109
artifactId: string,
119110
version: string,
120-
url: string,
121-
): Promise<PomData | null> {
111+
baseUrl?: string,
112+
): string {
113+
const groupPath = groupId.replace(/\./g, '/')
114+
return `${baseUrl ?? resolveRegistryBaseUrl(groupId)}/${groupPath}/${artifactId}/${version}/${artifactId}-${version}.pom`
115+
}
116+
117+
// prettier-ignore
118+
export async function fetchPom(groupId: string, artifactId: string, version: string, url: string): Promise<PomData | null> {
122119
try {
123120
const data = await getWithRetry(url)
124121
const parsed = parser.parse(data)
@@ -160,6 +157,7 @@ export async function fetchPom(
160157
const POM_CACHE_MAX_ENTRIES = 5_000
161158

162159
const pomCache = new Map<string, PomData>()
160+
// prettier-ignore
163161
const inFlight = new Map<string, Promise<PomData | null>>()
164162
const pomCacheStats = { hits: 0, coalesced: 0, misses: 0, evictions: 0 }
165163

@@ -186,11 +184,8 @@ function cacheSet(key: string, pom: PomData): void {
186184
* await it instead of issuing a duplicate request.
187185
* - Miss → performs the network fetch; caches the result only if non-null.
188186
*/
189-
async function fetchPomCached(
190-
groupId: string,
191-
artifactId: string,
192-
version: string,
193-
): Promise<PomData | null> {
187+
// prettier-ignore
188+
async function fetchPomCached(groupId: string, artifactId: string, version: string, baseUrl?: string): Promise<PomData | null> {
194189
const key = pomCacheKey(groupId, artifactId, version)
195190

196191
const cached = pomCache.get(key)
@@ -208,7 +203,12 @@ async function fetchPomCached(
208203
}
209204

210205
pomCacheStats.misses++
211-
const promise = fetchPom(groupId, artifactId, version, buildPomUrl(groupId, artifactId, version))
206+
const promise = fetchPom(
207+
groupId,
208+
artifactId,
209+
version,
210+
buildPomUrl(groupId, artifactId, version, baseUrl),
211+
)
212212
.then((pom) => {
213213
if (pom) cacheSet(key, pom)
214214
return pom
@@ -264,14 +264,9 @@ interface ResolvedFields {
264264
hops: number
265265
}
266266

267-
async function resolveWithInheritance(
268-
groupId: string,
269-
artifactId: string,
270-
version: string,
271-
depth = 0,
272-
visited = new Set<string>(),
273-
): Promise<ResolvedFields> {
274-
const pom = await fetchPomCached(groupId, artifactId, version)
267+
// prettier-ignore
268+
async function resolveWithInheritance(groupId: string, artifactId: string, version: string, depth = 0, visited = new Set<string>(), baseUrl?: string): Promise<ResolvedFields> {
269+
const pom = await fetchPomCached(groupId, artifactId, version, baseUrl)
275270
if (!pom) return emptyFields(depth)
276271

277272
const licenses = extractLicenses(pom)
@@ -299,6 +294,7 @@ async function resolveWithInheritance(
299294
parent.version,
300295
depth + 1,
301296
visited,
297+
resolveRegistryBaseUrl(parent.groupId),
302298
)
303299
return {
304300
description: extractStr(pom.description) ?? parentFields.description,
@@ -333,14 +329,11 @@ async function resolveWithInheritance(
333329
* Currently unused: kept as a lightweight option for high-throughput paths that
334330
* don't need parent inheritance.
335331
*/
336-
export async function extractArtifactDirect(
337-
groupId: string,
338-
artifactId: string,
339-
version: string,
340-
): Promise<PomExtractionResult> {
332+
// prettier-ignore
333+
export async function extractArtifactDirect(groupId: string, artifactId: string, version: string, baseUrl?: string): Promise<PomExtractionResult> {
341334
const purl = `pkg:maven/${groupId}/${artifactId}@${version}`
342-
const pomUrl = buildPomUrl(groupId, artifactId, version)
343-
const pom = await fetchPomCached(groupId, artifactId, version)
335+
const pomUrl = buildPomUrl(groupId, artifactId, version, baseUrl)
336+
const pom = await fetchPomCached(groupId, artifactId, version, baseUrl)
344337

345338
if (!pom) {
346339
return {
@@ -387,15 +380,12 @@ export async function extractArtifactDirect(
387380
* the parent chain to inherit licenses and SCM when not in the direct POM.
388381
* Always returns a result object; errors are captured in `result.error`.
389382
*/
390-
export async function extractArtifact(
391-
groupId: string,
392-
artifactId: string,
393-
version: string,
394-
): Promise<PomExtractionResult> {
383+
// prettier-ignore
384+
export async function extractArtifact(groupId: string, artifactId: string, version: string, baseUrl?: string): Promise<PomExtractionResult> {
395385
const purl = `pkg:maven/${groupId}/${artifactId}@${version}`
396386

397-
const pomUrl = buildPomUrl(groupId, artifactId, version)
398-
const rootPom = await fetchPomCached(groupId, artifactId, version)
387+
const pomUrl = buildPomUrl(groupId, artifactId, version, baseUrl)
388+
const rootPom = await fetchPomCached(groupId, artifactId, version, baseUrl)
399389
if (!rootPom) {
400390
return {
401391
groupId,
@@ -415,7 +405,14 @@ export async function extractArtifact(
415405
}
416406

417407
try {
418-
const resolved = await resolveWithInheritance(groupId, artifactId, version)
408+
const resolved = await resolveWithInheritance(
409+
groupId,
410+
artifactId,
411+
version,
412+
0,
413+
new Set(),
414+
baseUrl,
415+
)
419416
return {
420417
groupId,
421418
artifactId,
@@ -499,7 +496,7 @@ function extractLicenses(pom: PomData): string[] {
499496
const raw = pom.licenses?.license
500497
if (!raw) return []
501498
const list = Array.isArray(raw) ? raw : [raw]
502-
return (list as Array<{ name?: unknown }>)
499+
return (list as { name?: unknown }[])
503500
.map((l) => extractStr(l?.name))
504501
.filter((n): n is string => n !== null)
505502
}

‎services/apps/packages_worker/src/maven/metadata.ts‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,8 @@ import axios from 'axios'
1212
import { XMLParser } from 'fast-xml-parser'
1313

1414
import { isPrerelease } from './normalize'
15+
import { resolveRegistryBaseUrl } from './registry'
1516

16-
function getMavenRepo(): string {
17-
return process.env.MAVEN_FETCHER_BASE_URL ?? 'https://repo1.maven.org/maven2'
18-
}
1917
const REQUEST_TIMEOUT_MS = 10_000
2018
const MAX_RETRIES = 3
2119
const RETRY_BASE_MS = 2_000
@@ -78,9 +76,10 @@ export function pickStableRelease(candidate: string | null, versions: string[]):
7876
export async function resolveVersionsList(
7977
groupId: string,
8078
artifactId: string,
79+
baseUrl?: string,
8180
): Promise<MavenVersionsMetadata | MavenFetchError> {
8281
const groupPath = groupId.replace(/\./g, '/')
83-
const url = `${getMavenRepo()}/${groupPath}/${artifactId}/maven-metadata.xml`
82+
const url = `${baseUrl ?? resolveRegistryBaseUrl(groupId)}/${groupPath}/${artifactId}/maven-metadata.xml`
8483

8584
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
8685
try {
@@ -132,8 +131,9 @@ export async function resolveVersionsList(
132131
export async function resolveLatestVersion(
133132
groupId: string,
134133
artifactId: string,
134+
baseUrl?: string,
135135
): Promise<string | null> {
136-
const meta = await resolveVersionsList(groupId, artifactId)
136+
const meta = await resolveVersionsList(groupId, artifactId, baseUrl)
137137
if (isMavenFetchError(meta)) return null
138138
return meta.releaseVersion
139139
}

0 commit comments

Comments
 (0)