Skip to content

Commit e189609

Browse files
committed
fix: simplify pr for deployment
Signed-off-by: Umberto Sgueglia <usgueglia@contractor.linuxfoundation.org>
1 parent 7242d82 commit e189609

12 files changed

Lines changed: 45 additions & 675 deletions

File tree

backend/.env.dist.local

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,4 @@ POM_FETCHER_NON_CRITICAL_CONCURRENCY=20
208208
POM_FETCHER_REFRESH_DAYS=1
209209
POM_FETCHER_GROUP_DELAY_MS=100
210210
POM_FETCHER_FORCE_FULL_EXTRACTION=true
211-
POM_FETCHER_MAVEN_BASE_URL=https://maven-central.storage-download.googleapis.com/maven2
212-
MAVEN_SYNC_SOURCE=both
213-
MAVEN_DELTA_API_URL=https://maven-fetcher-production.up.railway.app
211+
POM_FETCHER_MAVEN_BASE_URL=https://maven-central.storage-download.googleapis.com/maven2

services/apps/packages_worker/src/config.ts

Lines changed: 0 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -46,44 +46,13 @@ export function getEnricherConfig() {
4646
}
4747
}
4848

49-
// Which source drives the critical Maven sync:
50-
// 'maven' → poll packages_universe by staleness (current behaviour, default/fallback)
51-
// 'api' → only enrich what our delta feed reports as changed
52-
// 'both' → run both passes in the same Temporal tick
53-
export type MavenSyncSource = 'api' | 'maven' | 'both'
54-
55-
function parseMavenSyncSource(raw: string | undefined): MavenSyncSource {
56-
if (raw === 'api' || raw === 'both') return raw
57-
// Anything else (unset, typo, legacy value) falls back to the current behaviour.
58-
return 'maven'
59-
}
60-
6149
export function getMavenConfig() {
62-
const syncSource = parseMavenSyncSource(process.env.MAVEN_SYNC_SOURCE)
63-
const deltaApiBaseUrl = (process.env.MAVEN_DELTA_API_URL ?? '').replace(/\/+$/, '')
64-
65-
if (syncSource !== 'maven' && !deltaApiBaseUrl) {
66-
throw new Error(`MAVEN_SYNC_SOURCE='${syncSource}' requires MAVEN_DELTA_API_URL to be set`)
67-
}
68-
6950
return {
7051
batchSize: requireEnvInt('POM_FETCHER_BATCH_SIZE'),
7152
concurrency: requireEnvInt('POM_FETCHER_CONCURRENCY'),
7253
nonCriticalBatchSize: requireEnvInt('POM_FETCHER_NON_CRITICAL_BATCH_SIZE'),
7354
nonCriticalConcurrency: requireEnvInt('POM_FETCHER_NON_CRITICAL_CONCURRENCY'),
7455
refreshDays: requireEnvInt('POM_FETCHER_REFRESH_DAYS'),
7556
groupDelayMs: requireEnvInt('POM_FETCHER_GROUP_DELAY_MS'),
76-
syncSource,
77-
deltaApi: {
78-
baseUrl: deltaApiBaseUrl,
79-
token: process.env.MAVEN_DELTA_API_TOKEN || undefined,
80-
pageSize: process.env.MAVEN_DELTA_API_PAGE_SIZE
81-
? parseInt(process.env.MAVEN_DELTA_API_PAGE_SIZE, 10)
82-
: 100,
83-
lookbackMinutes: process.env.MAVEN_DELTA_API_LOOKBACK_MINUTES
84-
? parseInt(process.env.MAVEN_DELTA_API_LOOKBACK_MINUTES, 10)
85-
: 15,
86-
includePrerelease: process.env.MAVEN_DELTA_API_INCLUDE_PRERELEASE === 'true',
87-
},
8857
}
8958
}

services/apps/packages_worker/src/maven/README.md

Lines changed: 0 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -247,25 +247,6 @@ so the worker throws on startup if any is missing. Suggested values shown.
247247
| `POM_FETCHER_REFRESH_DAYS` | `1` | Staleness window — re-sync a package once its `last_synced_at` is older than N days (applies to both phases) |
248248
| `POM_FETCHER_GROUP_DELAY_MS` | `200``400` | Delay between concurrent groups in the critical phase (rate-limit mitigation) |
249249

250-
### Sync source (Temporal critical path)
251-
252-
These select **where the critical sync gets its work from**. They affect only the Temporal
253-
`processMavenCriticalBatch` activity — the standalone backfill loop is unaffected. All are
254-
optional; unset/invalid values fall back to the current universe-polling behaviour.
255-
256-
| Env var | Default | Description |
257-
| ------------------------------------ | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- |
258-
| `MAVEN_SYNC_SOURCE` | `maven` | `maven` = poll `packages_universe` by staleness (current behaviour). `api` = enrich only what the delta feed reports. `both` = run both passes per tick. |
259-
| `MAVEN_DELTA_API_URL` || Base URL of our delta feed (e.g. the Railway deployment). **Required** when source is `api` or `both`. |
260-
| `MAVEN_DELTA_API_TOKEN` || Optional bearer token for the delta feed. |
261-
| `MAVEN_DELTA_API_PAGE_SIZE` | `100` | Page size for `/api/changes` pagination. |
262-
| `MAVEN_DELTA_API_LOOKBACK_MINUTES` | `15` | Rolling window size: each tick fetches `[now-N, now)`. Overlaps the cron interval on purpose — re-processing is safe (idempotent upserts). |
263-
| `MAVEN_DELTA_API_INCLUDE_PRERELEASE` | `false` | Forwarded as `includePrerelease` to the feed. |
264-
265-
The delta-API path always runs **full extraction** (the feed is an explicit "this changed"
266-
signal) and only enriches packages that are `is_critical` in `packages_universe`; non-critical
267-
purls in the feed are dropped.
268-
269250
**Concurrency guidance:** Maven Central handles 10–15 concurrent requests per IP without throttling. Retry logic with exponential backoff handles 429/403s. Keep `POM_FETCHER_CONCURRENCY` ≤ 5 locally — repeated local runs heat the IP (see [Known Exceptions](#maven-central-403-rate-limiting)).
270251

271252
---

services/apps/packages_worker/src/maven/activities.ts

Lines changed: 5 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -3,54 +3,18 @@ import { getServiceChildLogger } from '@crowd/logging'
33
import { getMavenConfig } from '../config'
44
import { getPackagesDb } from '../db'
55

6-
import { BatchResult, processApiChangesBatch, processBatch } from './runMavenEnrichmentLoop'
6+
import { BatchResult, processBatch } from './runMavenEnrichmentLoop'
77

88
const log = getServiceChildLogger('maven-activity')
99

10-
function addBatchResult(into: BatchResult, from: BatchResult): void {
11-
into.processed += from.processed
12-
into.skipped += from.skipped
13-
into.error += from.error
14-
into.unchanged += from.unchanged
15-
into.hopLimitReached += from.hopLimitReached
16-
}
17-
1810
export async function processMavenCriticalBatch(): Promise<BatchResult> {
1911
const config = getMavenConfig()
2012
const qx = await getPackagesDb()
2113

22-
const total: BatchResult = {
23-
processed: 0,
24-
skipped: 0,
25-
error: 0,
26-
unchanged: 0,
27-
hopLimitReached: 0,
28-
}
29-
30-
// Delta-API pass: enrich what our feed reports as changed (forces full extraction).
31-
if (config.syncSource === 'api' || config.syncSource === 'both') {
32-
try {
33-
const apiResult = await processApiChangesBatch(qx, config)
34-
log.info({ ...apiResult }, 'Maven delta-API batch complete')
35-
addBatchResult(total, apiResult)
36-
} catch (err) {
37-
// In 'both' mode the universe-polling pass is the reliable backbone, so a flaky
38-
// delta feed must never block it — log and continue. In 'api' mode there is no
39-
// fallback, so let the activity fail and have Temporal retry it.
40-
if (config.syncSource === 'api') throw err
41-
const message = err instanceof Error ? err.message : String(err)
42-
log.warn({ error: message }, 'Delta-API pass failed — continuing with universe-polling pass')
43-
}
44-
}
45-
46-
// Universe-polling pass: current behaviour — skip POM extraction when version is unchanged.
47-
if (config.syncSource === 'maven' || config.syncSource === 'both') {
48-
const mavenResult = await processBatch(qx, config, true, false)
49-
log.info({ ...mavenResult }, 'Maven critical batch complete')
50-
addBatchResult(total, mavenResult)
51-
}
52-
53-
return total
14+
// Universe-polling pass: skip POM extraction when version is unchanged.
15+
const result = await processBatch(qx, config, true, false)
16+
log.info({ ...result }, 'Maven critical batch complete')
17+
return result
5418
}
5519

5620
export async function processMavenNonCriticalBatch(): Promise<BatchResult> {

services/apps/packages_worker/src/maven/data_quality.sql

Lines changed: 0 additions & 166 deletions
This file was deleted.

0 commit comments

Comments
 (0)