Skip to content

Commit 4e22c9e

Browse files
ulemonsulemons
committed
fix: refactor and prevent daily to override the manual import
Signed-off-by: Umberto Sgueglia <usgueglia@contractor.linuxfoundation.org>
1 parent 6fb637c commit 4e22c9e

4 files changed

Lines changed: 98 additions & 134 deletions

File tree

‎services/apps/packages_worker/src/maven/scripts/csv.ts‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
export function parseCsv(content: string): Record<string, string>[] {
2+
const rows: string[][] = []
3+
let cur = ''
4+
let inQuote = false
5+
let row: string[] = []
6+
7+
for (let i = 0; i < content.length; i++) {
8+
const ch = content[i]
9+
const next = content[i + 1]
10+
11+
if (inQuote) {
12+
if (ch === '"' && next === '"') {
13+
cur += '"'
14+
i++
15+
} else if (ch === '"') {
16+
inQuote = false
17+
} else {
18+
cur += ch
19+
}
20+
} else {
21+
if (ch === '"') {
22+
inQuote = true
23+
} else if (ch === ',') {
24+
row.push(cur)
25+
cur = ''
26+
} else if (ch === '\r' && next === '\n') {
27+
row.push(cur)
28+
cur = ''
29+
rows.push(row)
30+
row = []
31+
i++
32+
} else if (ch === '\n') {
33+
row.push(cur)
34+
cur = ''
35+
rows.push(row)
36+
row = []
37+
} else {
38+
cur += ch
39+
}
40+
}
41+
}
42+
if (cur || row.length) {
43+
row.push(cur)
44+
rows.push(row)
45+
}
46+
47+
if (rows.length === 0) return []
48+
const headers = rows[0]
49+
return rows.slice(1).map((r) => {
50+
const obj: Record<string, string> = {}
51+
headers.forEach((h, i) => {
52+
obj[h.trim()] = (r[i] ?? '').trim()
53+
})
54+
return obj
55+
})
56+
}

‎services/apps/packages_worker/src/maven/scripts/fetchMavenMaintainers.ts‎

Lines changed: 24 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ import { extractArtifact, normalizeScmUrl } from '../extract'
2525
import { resolveLatestVersion } from '../metadata'
2626
import { resolveRegistryBaseUrl } from '../registry'
2727

28+
import { parseCsv } from './csv'
29+
2830
// ─── Config ───────────────────────────────────────────────────────────────────
2931

3032
const LIBRARIES_IO_KEY = process.env.LIBRARIES_IO_API_KEY ?? ''
@@ -84,83 +86,31 @@ async function safeGet<T>(
8486
url: string,
8587
params?: Record<string, string>,
8688
): Promise<T | null> {
87-
try {
88-
const res = await client.get<T>(url, { params })
89-
return res.data
90-
} catch (err) {
91-
if (axios.isAxiosError(err)) {
92-
const status = err.response?.status
93-
if (status === 404 || status === 422) return null
94-
if (status === 403 || status === 429) {
95-
const reset = err.response?.headers?.['x-ratelimit-reset']
96-
const wait = reset ? Math.max(0, Number(reset) * 1000 - Date.now()) + 1000 : 60_000
97-
console.warn(` Rate limited — waiting ${Math.round(wait / 1000)}s`)
98-
await sleep(wait)
99-
return safeGet<T>(client, url, params)
89+
const MAX_RETRIES = 5
90+
for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
91+
try {
92+
const res = await client.get<T>(url, { params })
93+
return res.data
94+
} catch (err) {
95+
if (axios.isAxiosError(err)) {
96+
const status = err.response?.status
97+
if (status === 404 || status === 422) return null
98+
if (status === 403 || status === 429) {
99+
const reset = err.response?.headers?.['x-ratelimit-reset']
100+
const wait = reset ? Math.max(0, Number(reset) * 1000 - Date.now()) + 1000 : 60_000
101+
console.warn(
102+
` Rate limited (attempt ${attempt + 1}/${MAX_RETRIES}) — waiting ${Math.round(wait / 1000)}s`,
103+
)
104+
await sleep(wait)
105+
continue
106+
}
107+
console.warn(` HTTP ${status} for ${url}`)
100108
}
101-
console.warn(` HTTP ${status} for ${url}`)
109+
return null
102110
}
103-
return null
104111
}
105-
}
106-
107-
// ─── CSV parser ───────────────────────────────────────────────────────────────
108-
109-
function parseCsv(content: string): Record<string, string>[] {
110-
const rows: string[][] = []
111-
let cur = ''
112-
let inQuote = false
113-
let row: string[] = []
114-
115-
for (let i = 0; i < content.length; i++) {
116-
const ch = content[i]
117-
const next = content[i + 1]
118-
119-
if (inQuote) {
120-
if (ch === '"' && next === '"') {
121-
cur += '"'
122-
i++
123-
} else if (ch === '"') {
124-
inQuote = false
125-
} else {
126-
cur += ch
127-
}
128-
} else {
129-
if (ch === '"') {
130-
inQuote = true
131-
} else if (ch === ',') {
132-
row.push(cur)
133-
cur = ''
134-
} else if (ch === '\r' && next === '\n') {
135-
row.push(cur)
136-
cur = ''
137-
rows.push(row)
138-
row = []
139-
i++
140-
} else if (ch === '\n') {
141-
row.push(cur)
142-
cur = ''
143-
rows.push(row)
144-
row = []
145-
} else {
146-
cur += ch
147-
}
148-
}
149-
}
150-
if (cur || row.length) {
151-
row.push(cur)
152-
rows.push(row)
153-
}
154-
155-
if (rows.length === 0) return []
156-
const headers = rows[0]
157-
return rows.slice(1).map((r) => {
158-
const obj: Record<string, string> = {}
159-
headers.forEach((h, i) => {
160-
obj[h.trim()] = (r[i] ?? '').trim()
161-
})
162-
return obj
163-
})
112+
console.warn(` Giving up after ${MAX_RETRIES} rate-limit retries: ${url}`)
113+
return null
164114
}
165115

166116
function csvEscape(value: string): string {

‎services/apps/packages_worker/src/maven/scripts/importMaintainersFromCsv.ts‎

Lines changed: 1 addition & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -26,64 +26,7 @@ import { pgpQx } from '@crowd/data-access-layer/src/queryExecutor'
2626

2727
import { getPackagesDbConnection } from '../../db'
2828

29-
// ─── CSV parser (same RFC-4180 parser used in fetchMavenMaintainers) ──────────
30-
31-
function parseCsv(content: string): Record<string, string>[] {
32-
const rows: string[][] = []
33-
let cur = ''
34-
let inQuote = false
35-
let row: string[] = []
36-
37-
for (let i = 0; i < content.length; i++) {
38-
const ch = content[i]
39-
const next = content[i + 1]
40-
41-
if (inQuote) {
42-
if (ch === '"' && next === '"') {
43-
cur += '"'
44-
i++
45-
} else if (ch === '"') {
46-
inQuote = false
47-
} else {
48-
cur += ch
49-
}
50-
} else {
51-
if (ch === '"') {
52-
inQuote = true
53-
} else if (ch === ',') {
54-
row.push(cur)
55-
cur = ''
56-
} else if (ch === '\r' && next === '\n') {
57-
row.push(cur)
58-
cur = ''
59-
rows.push(row)
60-
row = []
61-
i++
62-
} else if (ch === '\n') {
63-
row.push(cur)
64-
cur = ''
65-
rows.push(row)
66-
row = []
67-
} else {
68-
cur += ch
69-
}
70-
}
71-
}
72-
if (cur || row.length) {
73-
row.push(cur)
74-
rows.push(row)
75-
}
76-
77-
if (rows.length === 0) return []
78-
const headers = rows[0]
79-
return rows.slice(1).map((r) => {
80-
const obj: Record<string, string> = {}
81-
headers.forEach((h, i) => {
82-
obj[h.trim()] = (r[i] ?? '').trim()
83-
})
84-
return obj
85-
})
86-
}
29+
import { parseCsv } from './csv'
8730

8831
// ─── Role normalisation ───────────────────────────────────────────────────────
8932

‎services/libs/data-access-layer/src/osspckgs/maintainers.ts‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -74,12 +74,26 @@ export async function replacePackageMaintainers(
7474
links: Array<Pick<IDbPackageMaintainerUpsert, 'maintainerId' | 'role'>>,
7575
): Promise<string[]> {
7676
const before: Array<{ maintainer_id: number; role: string | null }> = await qx.select(
77-
`SELECT maintainer_id, role FROM package_maintainers WHERE package_id = $(packageId)`,
77+
`SELECT maintainer_id, role FROM package_maintainers
78+
WHERE package_id = $(packageId) AND ingestion_source IS DISTINCT FROM 'manual_csv'`,
7879
{ packageId },
7980
)
8081
const beforeMap = new Map(before.map((r) => [r.maintainer_id, r.role]))
8182

82-
await qx.result(`DELETE FROM package_maintainers WHERE package_id = $(packageId)`, { packageId })
83+
await qx.result(
84+
`DELETE FROM package_maintainers
85+
WHERE package_id = $(packageId) AND ingestion_source IS DISTINCT FROM 'manual_csv'`,
86+
{ packageId },
87+
)
88+
89+
// Surviving manual_csv links must be excluded from the changedFields diff —
90+
// they are pre-existing and should not appear as "added" in afterMap comparisons.
91+
const manualCsvRows: Array<{ maintainer_id: number }> = await qx.select(
92+
`SELECT maintainer_id FROM package_maintainers
93+
WHERE package_id = $(packageId) AND ingestion_source = 'manual_csv'`,
94+
{ packageId },
95+
)
96+
const manualCsvIds = new Set(manualCsvRows.map((r) => r.maintainer_id))
8397

8498
const afterMap = new Map<number, string | null>()
8599
for (const { maintainerId, role } of links) {
@@ -97,6 +111,7 @@ export async function replacePackageMaintainers(
97111
if (!afterMap.has(id)) changed.add('package_maintainers.maintainer_id')
98112
}
99113
for (const [id, role] of afterMap) {
114+
if (manualCsvIds.has(id)) continue
100115
if (!beforeMap.has(id)) changed.add('package_maintainers.maintainer_id')
101116
else if (beforeMap.get(id) !== role) changed.add('package_maintainers.role')
102117
}

0 commit comments

Comments
 (0)