Skip to content

Commit 7dd4205

Browse files
themaroltthemarolt
committed
Merge remote-tracking branch 'origin/main' into fix/incremental-depsdev-ingestion-CM-1281
2 parents f8c4463 + c8d1b5e commit 7dd4205

5 files changed

Lines changed: 487 additions & 98 deletions

File tree

‎services/apps/packages_worker/src/maven/__tests__/registry.test.ts‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
import { describe, expect, it } from 'vitest'
2+
3+
import {
4+
MAVEN_CENTRAL_BASE_URL,
5+
resolveRegistryBaseUrl,
6+
resolveRegistryPageUrl,
7+
resolveRegistryPageUrlFromBase,
8+
} from '../registry'
9+
10+
describe('resolveRegistryBaseUrl', () => {
11+
it('returns Google Maven for androidx namespace', () => {
12+
expect(resolveRegistryBaseUrl('androidx.annotation')).toBe(
13+
'https://dl.google.com/dl/android/maven2',
14+
)
15+
})
16+
17+
it('returns Google Maven for com.google.android.gms sub-namespace', () => {
18+
expect(resolveRegistryBaseUrl('com.google.android.gms')).toBe(
19+
'https://dl.google.com/dl/android/maven2',
20+
)
21+
})
22+
23+
it('returns Google Maven for bare com.google.android (legacy Android SDK stubs)', () => {
24+
expect(resolveRegistryBaseUrl('com.google.android')).toBe(
25+
'https://dl.google.com/dl/android/maven2',
26+
)
27+
})
28+
29+
it('does not match com.google.androidx — prefix boundary must end at a dot', () => {
30+
// com.google.android prefix must not bleed into unrelated com.google.android* strings
31+
expect(resolveRegistryBaseUrl('com.google.androidx')).toBe(MAVEN_CENTRAL_BASE_URL)
32+
})
33+
34+
it('returns Google Maven for android.arch (pre-AndroidX Architecture Components)', () => {
35+
expect(resolveRegistryBaseUrl('android.arch.core')).toBe(
36+
'https://dl.google.com/dl/android/maven2',
37+
)
38+
})
39+
40+
it('returns Google Maven for com.android.support (pre-AndroidX support library)', () => {
41+
expect(resolveRegistryBaseUrl('com.android.support')).toBe(
42+
'https://dl.google.com/dl/android/maven2',
43+
)
44+
})
45+
46+
it('returns Google Maven for com.google.testing.platform', () => {
47+
expect(resolveRegistryBaseUrl('com.google.testing.platform')).toBe(
48+
'https://dl.google.com/dl/android/maven2',
49+
)
50+
})
51+
52+
it('returns Google Maven for com.google.firebase namespace', () => {
53+
expect(resolveRegistryBaseUrl('com.google.firebase')).toBe(
54+
'https://dl.google.com/dl/android/maven2',
55+
)
56+
})
57+
58+
it('returns Google Maven for com.google.mlkit namespace', () => {
59+
expect(resolveRegistryBaseUrl('com.google.mlkit')).toBe(
60+
'https://dl.google.com/dl/android/maven2',
61+
)
62+
})
63+
64+
it('returns Gradle Plugin Portal for gradle.plugin namespace', () => {
65+
expect(resolveRegistryBaseUrl('gradle.plugin.name.remal')).toBe('https://plugins.gradle.org/m2')
66+
})
67+
68+
it('returns Jenkins repo for org.kohsuke.stapler (Jenkins HTTP framework)', () => {
69+
expect(resolveRegistryBaseUrl('org.kohsuke.stapler')).toBe('https://repo.jenkins-ci.org/public')
70+
})
71+
72+
it('returns Jenkins repo for org.jenkins-ci namespace', () => {
73+
expect(resolveRegistryBaseUrl('org.jenkins-ci.main')).toBe('https://repo.jenkins-ci.org/public')
74+
})
75+
76+
it('returns Jenkins repo for io.jenkins.plugins namespace', () => {
77+
expect(resolveRegistryBaseUrl('io.jenkins.plugins')).toBe('https://repo.jenkins-ci.org/public')
78+
})
79+
80+
it('returns Jenkins repo for io.jenkins.blueocean namespace', () => {
81+
expect(resolveRegistryBaseUrl('io.jenkins.blueocean')).toBe(
82+
'https://repo.jenkins-ci.org/public',
83+
)
84+
})
85+
86+
it('returns Maven Central for io.jenkins.tools (publishes on Central, not Jenkins repo)', () => {
87+
expect(resolveRegistryBaseUrl('io.jenkins.tools')).toBe('https://repo1.maven.org/maven2')
88+
})
89+
90+
it('returns Maven Central for unknown namespace', () => {
91+
expect(resolveRegistryBaseUrl('org.apache.commons')).toBe('https://repo1.maven.org/maven2')
92+
})
93+
94+
it('returns Google Maven for com.android.tools.analytics-library (Android Studio analytics)', () => {
95+
expect(resolveRegistryBaseUrl('com.android.tools.analytics-library')).toBe(
96+
'https://dl.google.com/dl/android/maven2',
97+
)
98+
})
99+
100+
it('returns Google Maven for com.android.tools.adblib (Android Debug Bridge Library)', () => {
101+
expect(resolveRegistryBaseUrl('com.android.tools.adblib')).toBe(
102+
'https://dl.google.com/dl/android/maven2',
103+
)
104+
})
105+
106+
it('returns Google Maven for com.android.tools.build (Android Gradle Plugin)', () => {
107+
expect(resolveRegistryBaseUrl('com.android.tools.build')).toBe(
108+
'https://dl.google.com/dl/android/maven2',
109+
)
110+
})
111+
112+
it('returns Google Maven for com.android.tools.build.jetifier (AndroidX migration tool)', () => {
113+
expect(resolveRegistryBaseUrl('com.android.tools.build.jetifier')).toBe(
114+
'https://dl.google.com/dl/android/maven2',
115+
)
116+
})
117+
118+
it('returns Google Maven for com.android.tools.utp (Android Unified Test Platform)', () => {
119+
expect(resolveRegistryBaseUrl('com.android.tools.utp')).toBe(
120+
'https://dl.google.com/dl/android/maven2',
121+
)
122+
})
123+
124+
it('returns Maven Central for bare com.android.tools (ddmlib, lint-api publish on Central)', () => {
125+
expect(resolveRegistryBaseUrl('com.android.tools')).toBe('https://repo1.maven.org/maven2')
126+
})
127+
128+
it('returns Google Maven for com.android.identity', () => {
129+
expect(resolveRegistryBaseUrl('com.android.identity')).toBe(
130+
'https://dl.google.com/dl/android/maven2',
131+
)
132+
})
133+
134+
it('returns Google Maven for com.google.mediapipe', () => {
135+
expect(resolveRegistryBaseUrl('com.google.mediapipe')).toBe(
136+
'https://dl.google.com/dl/android/maven2',
137+
)
138+
})
139+
140+
it('returns Jenkins repo for org.jenkinsci.plugins', () => {
141+
expect(resolveRegistryBaseUrl('org.jenkinsci.plugins')).toBe(
142+
'https://repo.jenkins-ci.org/public',
143+
)
144+
})
145+
146+
it('returns Jenkins repo for com.cloudbees.plugins', () => {
147+
expect(resolveRegistryBaseUrl('com.cloudbees.plugins')).toBe(
148+
'https://repo.jenkins-ci.org/public',
149+
)
150+
})
151+
})
152+
153+
describe('resolveRegistryPageUrl', () => {
154+
it('returns Google Maven browse URL for androidx', () => {
155+
expect(resolveRegistryPageUrl('androidx.annotation', 'annotation')).toBe(
156+
'https://maven.google.com/web/index.html#androidx.annotation:annotation',
157+
)
158+
})
159+
160+
it('returns Gradle Plugin Portal URL for gradle.plugin', () => {
161+
expect(resolveRegistryPageUrl('gradle.plugin.name.remal', 'gradle-plugins')).toBe(
162+
'https://plugins.gradle.org/m2/gradle/plugin/name/remal/gradle-plugins/',
163+
)
164+
})
165+
166+
it('returns Jenkins browse URL for org.jenkins-ci', () => {
167+
expect(resolveRegistryPageUrl('org.jenkins-ci.main', 'jenkins-core')).toBe(
168+
'https://repo.jenkins-ci.org/public/org/jenkins-ci/main/jenkins-core/',
169+
)
170+
})
171+
172+
it('returns Maven Central URL for unknown namespace', () => {
173+
expect(resolveRegistryPageUrl('org.apache.commons', 'commons-lang3')).toBe(
174+
'https://central.sonatype.com/artifact/org.apache.commons/commons-lang3',
175+
)
176+
})
177+
})
178+
179+
describe('resolveRegistryPageUrlFromBase', () => {
180+
it('returns Sonatype Central URL when resolvedBaseUrl is Maven Central', () => {
181+
expect(
182+
resolveRegistryPageUrlFromBase(
183+
'com.google.firebase',
184+
'firebase-admin',
185+
MAVEN_CENTRAL_BASE_URL,
186+
),
187+
).toBe('https://central.sonatype.com/artifact/com.google.firebase/firebase-admin')
188+
})
189+
190+
it('returns alternative registry page URL when resolvedBaseUrl is not Central', () => {
191+
const googleMavenUrl = 'https://dl.google.com/dl/android/maven2'
192+
expect(
193+
resolveRegistryPageUrlFromBase('com.google.firebase', 'firebase-analytics', googleMavenUrl),
194+
).toBe('https://maven.google.com/web/index.html#com.google.firebase:firebase-analytics')
195+
})
196+
})

‎services/apps/packages_worker/src/maven/extract.ts‎

Lines changed: 43 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ import { XMLParser } from 'fast-xml-parser'
77

88
import { getServiceChildLogger } from '@crowd/logging'
99

10+
import { resolveRegistryBaseUrl } from './registry'
11+
1012
const log = getServiceChildLogger('maven')
1113

1214
// ─── Types ────────────────────────────────────────────────────────────────────
@@ -56,14 +58,6 @@ interface PomPerson {
5658

5759
// ─── Config ───────────────────────────────────────────────────────────────────
5860

59-
// Lazy getter — evaluated at call time so the entry point can set MAVEN_FETCHER_BASE_URL
60-
// before the first HTTP request without worrying about module load order.
61-
// Backfill sets MAVEN_FETCHER_BASE_URL from MAVEN_FETCHER_BASE_URL_BACKFILL (GCS mirror);
62-
// the incremental Temporal worker sets it from MAVEN_FETCHER_BASE_URL_INCREMENTAL (repo1).
63-
function getMavenRepo(): string {
64-
return process.env.MAVEN_FETCHER_BASE_URL ?? 'https://repo1.maven.org/maven2'
65-
}
66-
6761
const REQUEST_TIMEOUT_MS = 15_000
6862

6963
const parser = new XMLParser({
@@ -78,10 +72,12 @@ const parser = new XMLParser({
7872
const MAX_RETRIES = 3
7973
const RETRY_BASE_MS = 2_000
8074

75+
// prettier-ignore
8176
async function sleep(ms: number): Promise<void> {
8277
return new Promise((r) => setTimeout(r, ms))
8378
}
8479

80+
// prettier-ignore
8581
async function getWithRetry(url: string): Promise<string> {
8682
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
8783
try {
@@ -108,17 +104,18 @@ async function getWithRetry(url: string): Promise<string> {
108104

109105
// ─── POM fetch ────────────────────────────────────────────────────────────────
110106

111-
export function buildPomUrl(groupId: string, artifactId: string, version: string): string {
112-
const groupPath = groupId.replace(/\./g, '/')
113-
return `${getMavenRepo()}/${groupPath}/${artifactId}/${version}/${artifactId}-${version}.pom`
114-
}
115-
116-
export async function fetchPom(
107+
export function buildPomUrl(
117108
groupId: string,
118109
artifactId: string,
119110
version: string,
120-
url: string,
121-
): Promise<PomData | null> {
111+
baseUrl?: string,
112+
): string {
113+
const groupPath = groupId.replace(/\./g, '/')
114+
return `${baseUrl ?? resolveRegistryBaseUrl(groupId)}/${groupPath}/${artifactId}/${version}/${artifactId}-${version}.pom`
115+
}
116+
117+
// prettier-ignore
118+
export async function fetchPom(groupId: string, artifactId: string, version: string, url: string): Promise<PomData | null> {
122119
try {
123120
const data = await getWithRetry(url)
124121
const parsed = parser.parse(data)
@@ -160,6 +157,7 @@ export async function fetchPom(
160157
const POM_CACHE_MAX_ENTRIES = 5_000
161158

162159
const pomCache = new Map<string, PomData>()
160+
// prettier-ignore
163161
const inFlight = new Map<string, Promise<PomData | null>>()
164162
const pomCacheStats = { hits: 0, coalesced: 0, misses: 0, evictions: 0 }
165163

@@ -186,11 +184,8 @@ function cacheSet(key: string, pom: PomData): void {
186184
* await it instead of issuing a duplicate request.
187185
* - Miss → performs the network fetch; caches the result only if non-null.
188186
*/
189-
async function fetchPomCached(
190-
groupId: string,
191-
artifactId: string,
192-
version: string,
193-
): Promise<PomData | null> {
187+
// prettier-ignore
188+
async function fetchPomCached(groupId: string, artifactId: string, version: string, baseUrl?: string): Promise<PomData | null> {
194189
const key = pomCacheKey(groupId, artifactId, version)
195190

196191
const cached = pomCache.get(key)
@@ -208,7 +203,12 @@ async function fetchPomCached(
208203
}
209204

210205
pomCacheStats.misses++
211-
const promise = fetchPom(groupId, artifactId, version, buildPomUrl(groupId, artifactId, version))
206+
const promise = fetchPom(
207+
groupId,
208+
artifactId,
209+
version,
210+
buildPomUrl(groupId, artifactId, version, baseUrl),
211+
)
212212
.then((pom) => {
213213
if (pom) cacheSet(key, pom)
214214
return pom
@@ -264,14 +264,9 @@ interface ResolvedFields {
264264
hops: number
265265
}
266266

267-
async function resolveWithInheritance(
268-
groupId: string,
269-
artifactId: string,
270-
version: string,
271-
depth = 0,
272-
visited = new Set<string>(),
273-
): Promise<ResolvedFields> {
274-
const pom = await fetchPomCached(groupId, artifactId, version)
267+
// prettier-ignore
268+
async function resolveWithInheritance(groupId: string, artifactId: string, version: string, depth = 0, visited = new Set<string>(), baseUrl?: string): Promise<ResolvedFields> {
269+
const pom = await fetchPomCached(groupId, artifactId, version, baseUrl)
275270
if (!pom) return emptyFields(depth)
276271

277272
const licenses = extractLicenses(pom)
@@ -299,6 +294,7 @@ async function resolveWithInheritance(
299294
parent.version,
300295
depth + 1,
301296
visited,
297+
resolveRegistryBaseUrl(parent.groupId),
302298
)
303299
return {
304300
description: extractStr(pom.description) ?? parentFields.description,
@@ -333,14 +329,11 @@ async function resolveWithInheritance(
333329
* Currently unused: kept as a lightweight option for high-throughput paths that
334330
* don't need parent inheritance.
335331
*/
336-
export async function extractArtifactDirect(
337-
groupId: string,
338-
artifactId: string,
339-
version: string,
340-
): Promise<PomExtractionResult> {
332+
// prettier-ignore
333+
export async function extractArtifactDirect(groupId: string, artifactId: string, version: string, baseUrl?: string): Promise<PomExtractionResult> {
341334
const purl = `pkg:maven/${groupId}/${artifactId}@${version}`
342-
const pomUrl = buildPomUrl(groupId, artifactId, version)
343-
const pom = await fetchPomCached(groupId, artifactId, version)
335+
const pomUrl = buildPomUrl(groupId, artifactId, version, baseUrl)
336+
const pom = await fetchPomCached(groupId, artifactId, version, baseUrl)
344337

345338
if (!pom) {
346339
return {
@@ -387,15 +380,12 @@ export async function extractArtifactDirect(
387380
* the parent chain to inherit licenses and SCM when not in the direct POM.
388381
* Always returns a result object; errors are captured in `result.error`.
389382
*/
390-
export async function extractArtifact(
391-
groupId: string,
392-
artifactId: string,
393-
version: string,
394-
): Promise<PomExtractionResult> {
383+
// prettier-ignore
384+
export async function extractArtifact(groupId: string, artifactId: string, version: string, baseUrl?: string): Promise<PomExtractionResult> {
395385
const purl = `pkg:maven/${groupId}/${artifactId}@${version}`
396386

397-
const pomUrl = buildPomUrl(groupId, artifactId, version)
398-
const rootPom = await fetchPomCached(groupId, artifactId, version)
387+
const pomUrl = buildPomUrl(groupId, artifactId, version, baseUrl)
388+
const rootPom = await fetchPomCached(groupId, artifactId, version, baseUrl)
399389
if (!rootPom) {
400390
return {
401391
groupId,
@@ -415,7 +405,14 @@ export async function extractArtifact(
415405
}
416406

417407
try {
418-
const resolved = await resolveWithInheritance(groupId, artifactId, version)
408+
const resolved = await resolveWithInheritance(
409+
groupId,
410+
artifactId,
411+
version,
412+
0,
413+
new Set(),
414+
baseUrl,
415+
)
419416
return {
420417
groupId,
421418
artifactId,
@@ -499,7 +496,7 @@ function extractLicenses(pom: PomData): string[] {
499496
const raw = pom.licenses?.license
500497
if (!raw) return []
501498
const list = Array.isArray(raw) ? raw : [raw]
502-
return (list as Array<{ name?: unknown }>)
499+
return (list as { name?: unknown }[])
503500
.map((l) => extractStr(l?.name))
504501
.filter((n): n is string => n !== null)
505502
}

0 commit comments

Comments
 (0)