Skip to content

Commit b49de68

Browse files
ulemonsulemons
committed
feat: pom extractor
Signed-off-by: Umberto Sgueglia <usgueglia@contractor.linuxfoundation.org>
1 parent c1c404c commit b49de68

13 files changed

Lines changed: 963 additions & 8 deletions

File tree

‎pnpm-lock.yaml‎

Lines changed: 74 additions & 8 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎scripts/services/pom-fetcher.yaml‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
version: '3.1'
2+
3+
x-env-args: &env-args
4+
DOCKER_BUILDKIT: 1
5+
NODE_ENV: docker
6+
SERVICE: pom-fetcher
7+
SHELL: /bin/sh
8+
SUPPRESS_NO_CONFIG_WARNING: 'true'
9+
POM_FETCHER_BATCH_SIZE: '200'
10+
POM_FETCHER_CONCURRENCY: '10'
11+
POM_FETCHER_STALE_DAYS: '7'
12+
POM_FETCHER_IDLE_SLEEP_SEC: '3600'
13+
14+
services:
15+
pom-fetcher:
16+
build:
17+
context: ../../
18+
dockerfile: ./scripts/services/docker/Dockerfile.packages-worker
19+
command: 'pnpm run start:pom-fetcher'
20+
working_dir: /usr/crowd/app/services/apps/packages_worker
21+
env_file:
22+
- ../../backend/.env.dist.local
23+
- ../../backend/.env.dist.composed
24+
- ../../backend/.env.override.local
25+
- ../../backend/.env.override.composed
26+
environment:
27+
<<: *env-args
28+
restart: always
29+
networks:
30+
- crowd-bridge
31+
32+
pom-fetcher-dev:
33+
build:
34+
context: ../../
35+
dockerfile: ./scripts/services/docker/Dockerfile.packages-worker
36+
command: 'pnpm run dev:pom-fetcher'
37+
working_dir: /usr/crowd/app/services/apps/packages_worker
38+
# user: '${USER_ID}:${GROUP_ID}'
39+
env_file:
40+
- ../../backend/.env.dist.local
41+
- ../../backend/.env.dist.composed
42+
- ../../backend/.env.override.local
43+
- ../../backend/.env.override.composed
44+
environment:
45+
<<: *env-args
46+
hostname: pom-fetcher
47+
networks:
48+
- crowd-bridge
49+
volumes:
50+
- ../../services/libs/audit-logs/src:/usr/crowd/app/services/libs/audit-logs/src
51+
- ../../services/libs/common/src:/usr/crowd/app/services/libs/common/src
52+
- ../../services/libs/common_services/src:/usr/crowd/app/services/libs/common_services/src
53+
- ../../services/libs/data-access-layer/src:/usr/crowd/app/services/libs/data-access-layer/src
54+
- ../../services/libs/database/src:/usr/crowd/app/services/libs/database/src
55+
- ../../services/libs/integrations/src:/usr/crowd/app/services/libs/integrations/src
56+
- ../../services/libs/logging/src:/usr/crowd/app/services/libs/logging/src
57+
- ../../services/libs/nango/src:/usr/crowd/app/services/libs/nango/src
58+
- ../../services/libs/opensearch/src:/usr/crowd/app/services/libs/opensearch/src
59+
- ../../services/libs/queue/src:/usr/crowd/app/services/libs/queue/src
60+
- ../../services/libs/redis/src:/usr/crowd/app/services/libs/redis/src
61+
- ../../services/libs/snowflake/src:/usr/crowd/app/services/libs/snowflake/src
62+
- ../../services/libs/telemetry/src:/usr/crowd/app/services/libs/telemetry/src
63+
- ../../services/libs/temporal/src:/usr/crowd/app/services/libs/temporal/src
64+
- ../../services/libs/types/src:/usr/crowd/app/services/libs/types/src
65+
- ../../services/apps/packages_worker/src:/usr/crowd/app/services/apps/packages_worker/src
66+
67+
networks:
68+
crowd-bridge:
69+
external: true

‎services/apps/packages_worker/package.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,13 @@
44
"scripts": {
55
"start:packages-worker": "SERVICE=packages-worker tsx src/bin/packages-worker.ts",
66
"start:github-repos-enricher": "SERVICE=github-repos-enricher tsx src/bin/github-repos-enricher.ts",
7+
"start:pom-fetcher": "SERVICE=pom-fetcher tsx src/bin/pom-fetcher.ts",
78
"dev:packages-worker": "SERVICE=packages-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9233 src/bin/packages-worker.ts",
89
"dev:github-repos-enricher": "SERVICE=github-repos-enricher LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9234 src/bin/github-repos-enricher.ts",
10+
"dev:pom-fetcher": "SERVICE=pom-fetcher LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9235 src/bin/pom-fetcher.ts",
911
"dev:packages-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=packages-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9233 src/bin/packages-worker.ts",
1012
"dev:github-repos-enricher:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=github-repos-enricher LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9234 src/bin/github-repos-enricher.ts",
13+
"dev:pom-fetcher:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=pom-fetcher LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9235 src/bin/pom-fetcher.ts",
1114
"lint": "npx eslint --ext .ts src --max-warnings=0",
1215
"format": "npx prettier --write \"src/**/*.ts\"",
1316
"format-check": "npx prettier --check .",
@@ -18,6 +21,8 @@
1821
"@crowd/data-access-layer": "workspace:*",
1922
"@crowd/database": "workspace:*",
2023
"@crowd/logging": "workspace:*",
24+
"axios": "^1.6.7",
25+
"fast-xml-parser": "^4.4.0",
2126
"tsx": "^4.7.1",
2227
"typescript": "^5.6.3"
2328
},

‎services/apps/packages_worker/src/bin/pom-fetcher.ts‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
import { getServiceLogger } from '@crowd/logging'
2+
3+
import { getPomFetcherConfig } from '../config'
4+
import { getPackagesDb } from '../db'
5+
import { runPomEnrichmentLoop } from '../pom-fetcher/runPomEnrichmentLoop'
6+
7+
const log = getServiceLogger()
8+
9+
let shuttingDown = false
10+
11+
const shutdown = async () => {
12+
if (shuttingDown) return
13+
shuttingDown = true
14+
log.info('Shutting down pom-fetcher...')
15+
}
16+
17+
process.on('SIGINT', shutdown)
18+
process.on('SIGTERM', shutdown)
19+
20+
const main = async () => {
21+
log.info('pom-fetcher starting...')
22+
23+
const config = getPomFetcherConfig()
24+
log.info(
25+
{ batchSize: config.batchSize, concurrency: config.concurrency, staleDays: config.staleDays },
26+
'Config loaded',
27+
)
28+
29+
const qx = await getPackagesDb()
30+
await qx.selectOne('SELECT 1')
31+
log.info('Connected to packages-db.')
32+
33+
await runPomEnrichmentLoop(qx, config, () => shuttingDown)
34+
35+
log.info('pom-fetcher stopped.')
36+
process.exit(0)
37+
}
38+
39+
main().catch((err) => {
40+
log.error({ err }, 'pom-fetcher fatal error')
41+
process.exit(1)
42+
})

‎services/apps/packages_worker/src/config.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,3 +29,12 @@ export function getEnricherConfig() {
2929
idleSleepSec: requireEnvInt('ENRICHER_IDLE_SLEEP_SEC'),
3030
}
3131
}
32+
33+
export function getPomFetcherConfig() {
34+
return {
35+
batchSize: parseInt(process.env.POM_FETCHER_BATCH_SIZE ?? '200', 10),
36+
concurrency: parseInt(process.env.POM_FETCHER_CONCURRENCY ?? '10', 10),
37+
staleDays: parseInt(process.env.POM_FETCHER_STALE_DAYS ?? '7', 10),
38+
idleSleepSec: parseInt(process.env.POM_FETCHER_IDLE_SLEEP_SEC ?? '3600', 10),
39+
}
40+
}

0 commit comments

Comments
 (0)