Skip to content

Commit 48593db

Browse files
joanagmaiajoanagmaia
authored
chore: osv-worker deployment (#4175)
Signed-off-by: Joana Maia <jmaia@contractor.linuxfoundation.org>
1 parent 661161e commit 48593db

5 files changed

Lines changed: 80 additions & 2 deletions

File tree

‎scripts/builders/packages.env‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
DOCKERFILE="./services/docker/Dockerfile.packages"
22
CONTEXT="../"
33
REPO="sjc.ocir.io/axbydjxa5zuh/packages"
4-
SERVICES="packages github-repos-enricher deps-dev-ingest npm-worker maven-worker"
4+
SERVICES="packages github-repos-enricher deps-dev-ingest npm-worker maven-worker osv-worker"

‎scripts/services/osv-worker.yaml‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
version: '3.1'
2+
3+
x-env-args: &env-args
4+
DOCKER_BUILDKIT: 1
5+
NODE_ENV: docker
6+
SERVICE: osv-worker
7+
CROWD_TEMPORAL_TASKQUEUE: osv-worker
8+
CROWD_TEMPORAL_NAMESPACE: ${CROWD_PACKAGES_TEMPORAL_NAMESPACE}
9+
SHELL: /bin/sh
10+
SUPPRESS_NO_CONFIG_WARNING: 'true'
11+
12+
services:
13+
osv-worker:
14+
build:
15+
context: ../../
16+
dockerfile: ./scripts/services/docker/Dockerfile.packages
17+
command: 'pnpm run start:osv-worker'
18+
working_dir: /usr/crowd/app/services/apps/packages_worker
19+
env_file:
20+
- ../../backend/.env.dist.local
21+
- ../../backend/.env.dist.composed
22+
- ../../backend/.env.override.local
23+
- ../../backend/.env.override.composed
24+
environment:
25+
<<: *env-args
26+
restart: always
27+
networks:
28+
- crowd-bridge
29+
30+
osv-worker-dev:
31+
build:
32+
context: ../../
33+
dockerfile: ./scripts/services/docker/Dockerfile.packages
34+
command: 'pnpm run dev:osv-worker'
35+
working_dir: /usr/crowd/app/services/apps/packages_worker
36+
# user: '${USER_ID}:${GROUP_ID}'
37+
env_file:
38+
- ../../backend/.env.dist.local
39+
- ../../backend/.env.dist.composed
40+
- ../../backend/.env.override.local
41+
- ../../backend/.env.override.composed
42+
environment:
43+
<<: *env-args
44+
hostname: osv-worker
45+
networks:
46+
- crowd-bridge
47+
volumes:
48+
- ../../services/libs/audit-logs/src:/usr/crowd/app/services/libs/audit-logs/src
49+
- ../../services/libs/common/src:/usr/crowd/app/services/libs/common/src
50+
- ../../services/libs/common_services/src:/usr/crowd/app/services/libs/common_services/src
51+
- ../../services/libs/data-access-layer/src:/usr/crowd/app/services/libs/data-access-layer/src
52+
- ../../services/libs/database/src:/usr/crowd/app/services/libs/database/src
53+
- ../../services/libs/integrations/src:/usr/crowd/app/services/libs/integrations/src
54+
- ../../services/libs/logging/src:/usr/crowd/app/services/libs/logging/src
55+
- ../../services/libs/nango/src:/usr/crowd/app/services/libs/nango/src
56+
- ../../services/libs/opensearch/src:/usr/crowd/app/services/libs/opensearch/src
57+
- ../../services/libs/queue/src:/usr/crowd/app/services/libs/queue/src
58+
- ../../services/libs/redis/src:/usr/crowd/app/services/libs/redis/src
59+
- ../../services/libs/snowflake/src:/usr/crowd/app/services/libs/snowflake/src
60+
- ../../services/libs/telemetry/src:/usr/crowd/app/services/libs/telemetry/src
61+
- ../../services/libs/temporal/src:/usr/crowd/app/services/libs/temporal/src
62+
- ../../services/libs/types/src:/usr/crowd/app/services/libs/types/src
63+
- ../../services/apps/packages_worker/src:/usr/crowd/app/services/apps/packages_worker/src
64+
65+
networks:
66+
crowd-bridge:
67+
external: true

‎services/apps/packages_worker/package.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
"start:criticality-worker": "CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=criticality-worker tsx src/bin/criticality-worker.ts",
77
"start:deps-dev-ingest": "CROWD_TEMPORAL_TASKQUEUE=deps-dev-ingest CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=deps-dev-ingest tsx src/bin/deps-dev-ingest.ts",
88
"start:npm-worker": "CROWD_TEMPORAL_TASKQUEUE=npm-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=npm-worker tsx src/bin/npm-worker.ts",
9+
"start:osv-worker": "CROWD_TEMPORAL_TASKQUEUE=osv-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=osv-worker tsx src/bin/osv-worker.ts",
910
"start:github-repos-enricher": "SERVICE=github-repos-enricher tsx src/bin/github-repos-enricher.ts",
1011
"run:pagerank": "tsx src/criticality/run-pagerank.ts",
1112
"run:impact": "tsx src/criticality/run-impact.ts",
@@ -19,12 +20,14 @@
1920
"dev:maven-worker": "CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=maven-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9236 src/bin/maven-worker.ts",
2021
"dev:deps-dev-ingest": "CROWD_TEMPORAL_TASKQUEUE=deps-dev-ingest CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=deps-dev-ingest nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9235 src/bin/deps-dev-ingest.ts",
2122
"dev:npm-worker": "CROWD_TEMPORAL_TASKQUEUE=npm-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=npm-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9236 src/bin/npm-worker.ts",
23+
"dev:osv-worker": "CROWD_TEMPORAL_TASKQUEUE=osv-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=osv-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9238 src/bin/osv-worker.ts",
2224
"dev:github-repos-enricher": "SERVICE=github-repos-enricher LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9234 src/bin/github-repos-enricher.ts",
2325
"dev:packages-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=packages-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9233 src/bin/packages-worker.ts",
2426
"dev:criticality-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=criticality-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9237 src/bin/criticality-worker.ts",
2527
"dev:maven-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=maven-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9236 src/bin/maven-worker.ts",
2628
"dev:deps-dev-ingest:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=deps-dev-ingest CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=deps-dev-ingest nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9235 src/bin/deps-dev-ingest.ts",
2729
"dev:npm-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=npm-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=npm-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9236 src/bin/npm-worker.ts",
30+
"dev:osv-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=osv-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=osv-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9238 src/bin/osv-worker.ts",
2831
"start:maven": "SERVICE=maven tsx src/bin/maven.ts",
2932
"dev:maven": "SERVICE=maven LOG_LEVEL=info nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9235 src/bin/maven.ts",
3033
"dev:github-repos-enricher:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=github-repos-enricher LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9234 src/bin/github-repos-enricher.ts",

‎services/apps/packages_worker/src/bin/osv-worker.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
import { scheduleOsvSync } from '../osv/schedule'
2+
import { svc } from '../service'
3+
4+
setImmediate(async () => {
5+
await svc.init()
6+
await scheduleOsvSync()
7+
await svc.start()
8+
})

‎services/apps/packages_worker/src/osv/schedule.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ export async function scheduleOsvSync(): Promise<void> {
5858
action: {
5959
type: 'startWorkflow',
6060
workflowType: osvSync,
61-
taskQueue: 'packages-worker',
61+
taskQueue: 'osv-worker',
6262
// Headroom for npm (~1 hour today) + Maven (~5 minutes) + derive
6363
// (~5 minutes for 600-700k packages); 4 hours leaves space for the
6464
// upsertOne N+1 deferred fix being slower than expected.

0 commit comments

Comments
 (0)