Skip to content

Commit 17c0bc9

Browse files
mbani01mbani01
authored
feat: security contacts worker [CM-1297] (#4283)
Signed-off-by: Mouad BANI <mouad-mb@outlook.com>
1 parent 1199eb5 commit 17c0bc9

37 files changed

Lines changed: 2276 additions & 64 deletions

‎backend/.env.dist.composed‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,3 +35,6 @@ CROWD_PACKAGES_DB_PORT=5432
3535
CROWD_PACKAGES_DB_USERNAME=postgres
3636
CROWD_PACKAGES_DB_PASSWORD=example
3737
CROWD_PACKAGES_DB_DATABASE=packages-db
38+
39+
# security-contacts-worker
40+
SECURITY_CONTACTS_USER_AGENT="lfx-security-contacts-worker"

‎backend/.env.dist.local‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,9 @@ ENRICHER_BATCH_SIZE=100
185185
ENRICHER_REPO_UPDATE_INTERVAL_HOURS=24
186186
ENRICHER_IDLE_SLEEP_SEC=60
187187

188+
# security-contacts-worker
189+
SECURITY_CONTACTS_USER_AGENT="lfx-security-contacts-worker"
190+
188191
OSSPCKGS_GCP_PROJECT=local-dev
189192
OSSPCKGS_GCS_BUCKET=local-dev
190193
OSSPCKGS_GCP_CREDENTIALS_B64=e30=

‎backend/src/osspckgs/migrations/V1782950400__security_contacts.sql‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
CREATE TABLE IF NOT EXISTS security_contacts (
2+
id BIGSERIAL PRIMARY KEY,
3+
repo_id BIGINT NOT NULL REFERENCES repos(id) ON DELETE CASCADE,
4+
channel TEXT NOT NULL,
5+
value TEXT NOT NULL,
6+
role TEXT NOT NULL,
7+
name TEXT,
8+
score NUMERIC(4,3) NOT NULL,
9+
confidence TEXT NOT NULL,
10+
provenance JSONB NOT NULL DEFAULT '[]',
11+
last_refreshed TIMESTAMPTZ NOT NULL DEFAULT NOW(),
12+
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
13+
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
14+
);
15+
16+
CREATE UNIQUE INDEX IF NOT EXISTS security_contacts_repo_channel_value_uq
17+
ON security_contacts (repo_id, channel, value);
18+
19+
CREATE INDEX IF NOT EXISTS security_contacts_repo_confidence_idx
20+
ON security_contacts (repo_id, confidence);
21+
22+
ALTER TABLE repos ADD COLUMN IF NOT EXISTS pvr_enabled BOOL;
23+
ALTER TABLE repos ADD COLUMN IF NOT EXISTS security_policy_url TEXT;
24+
ALTER TABLE repos ADD COLUMN IF NOT EXISTS vulnerability_reporting_url TEXT;
25+
ALTER TABLE repos ADD COLUMN IF NOT EXISTS bug_bounty_url TEXT;
26+
ALTER TABLE repos ADD COLUMN IF NOT EXISTS security_txt_url TEXT;
27+
ALTER TABLE repos ADD COLUMN IF NOT EXISTS contacts_last_refreshed TIMESTAMPTZ;

‎pnpm-lock.yaml‎

Lines changed: 7 additions & 9 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎scripts/builders/packages.env‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
DOCKERFILE="./services/docker/Dockerfile.packages"
22
CONTEXT="../"
33
REPO="sjc.ocir.io/axbydjxa5zuh/packages"
4-
SERVICES="github-repos-enricher bq-dataset-ingest npm-worker maven-worker osv-worker dockerhub-sync cargo-worker go-worker nuget-worker"
4+
SERVICES="github-repos-enricher bq-dataset-ingest npm-worker maven-worker osv-worker dockerhub-sync cargo-worker go-worker nuget-worker security-contacts-worker"

‎scripts/services/security-contacts-worker.yaml‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
version: '3.1'
2+
3+
x-env-args: &env-args
4+
DOCKER_BUILDKIT: 1
5+
NODE_ENV: docker
6+
SERVICE: security-contacts-worker
7+
SHELL: /bin/sh
8+
SUPPRESS_NO_CONFIG_WARNING: 'true'
9+
CROWD_TEMPORAL_TASKQUEUE: packages-worker
10+
11+
services:
12+
security-contacts-worker:
13+
build:
14+
context: ../../
15+
dockerfile: ./scripts/services/docker/Dockerfile.packages
16+
command: 'pnpm run start:security-contacts-worker'
17+
working_dir: /usr/crowd/app/services/apps/packages_worker
18+
env_file:
19+
- ../../backend/.env.dist.local
20+
- ../../backend/.env.dist.composed
21+
- ../../backend/.env.override.local
22+
- ../../backend/.env.override.composed
23+
environment:
24+
<<: *env-args
25+
restart: always
26+
networks:
27+
- crowd-bridge
28+
29+
security-contacts-worker-dev:
30+
build:
31+
context: ../../
32+
dockerfile: ./scripts/services/docker/Dockerfile.packages
33+
command: 'pnpm run dev:security-contacts-worker'
34+
working_dir: /usr/crowd/app/services/apps/packages_worker
35+
env_file:
36+
- ../../backend/.env.dist.local
37+
- ../../backend/.env.dist.composed
38+
- ../../backend/.env.override.local
39+
- ../../backend/.env.override.composed
40+
environment:
41+
<<: *env-args
42+
hostname: security-contacts-worker
43+
networks:
44+
- crowd-bridge
45+
volumes:
46+
- ../../services/libs/audit-logs/src:/usr/crowd/app/services/libs/audit-logs/src
47+
- ../../services/libs/common/src:/usr/crowd/app/services/libs/common/src
48+
- ../../services/libs/common_services/src:/usr/crowd/app/services/libs/common_services/src
49+
- ../../services/libs/data-access-layer/src:/usr/crowd/app/services/libs/data-access-layer/src
50+
- ../../services/libs/database/src:/usr/crowd/app/services/libs/database/src
51+
- ../../services/libs/integrations/src:/usr/crowd/app/services/libs/integrations/src
52+
- ../../services/libs/logging/src:/usr/crowd/app/services/libs/logging/src
53+
- ../../services/libs/nango/src:/usr/crowd/app/services/libs/nango/src
54+
- ../../services/libs/opensearch/src:/usr/crowd/app/services/libs/opensearch/src
55+
- ../../services/libs/queue/src:/usr/crowd/app/services/libs/queue/src
56+
- ../../services/libs/redis/src:/usr/crowd/app/services/libs/redis/src
57+
- ../../services/libs/snowflake/src:/usr/crowd/app/services/libs/snowflake/src
58+
- ../../services/libs/telemetry/src:/usr/crowd/app/services/libs/telemetry/src
59+
- ../../services/libs/temporal/src:/usr/crowd/app/services/libs/temporal/src
60+
- ../../services/libs/types/src:/usr/crowd/app/services/libs/types/src
61+
- ../../services/apps/packages_worker/src:/usr/crowd/app/services/apps/packages_worker/src
62+
63+
networks:
64+
crowd-bridge:
65+
external: true

‎services/apps/packages_worker/package.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,9 @@
4040
"start:go-worker": "CROWD_TEMPORAL_TASKQUEUE=go-worker SERVICE=go-worker tsx src/bin/go-worker.ts",
4141
"dev:go-worker": "CROWD_TEMPORAL_TASKQUEUE=go-worker SERVICE=go-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9241 src/bin/go-worker.ts",
4242
"dev:go-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=go-worker SERVICE=go-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9241 src/bin/go-worker.ts",
43+
"start:security-contacts-worker": "CROWD_TEMPORAL_TASKQUEUE=packages-worker SERVICE=security-contacts-worker tsx src/bin/security-contacts-worker.ts",
44+
"dev:security-contacts-worker": "CROWD_TEMPORAL_TASKQUEUE=packages-worker SERVICE=security-contacts-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9243 src/bin/security-contacts-worker.ts",
45+
"dev:security-contacts-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=packages-worker SERVICE=security-contacts-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9243 src/bin/security-contacts-worker.ts",
4346
"start:nuget-worker": "CROWD_TEMPORAL_TASKQUEUE=nuget-worker SERVICE=nuget-worker tsx src/bin/nuget-worker.ts",
4447
"dev:nuget-worker": "CROWD_TEMPORAL_TASKQUEUE=nuget-worker SERVICE=nuget-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9242 src/bin/nuget-worker.ts",
4548
"dev:nuget-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=nuget-worker SERVICE=nuget-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9242 src/bin/nuget-worker.ts",
@@ -72,6 +75,7 @@
7275
"@temporalio/activity": "~1.17.2",
7376
"@temporalio/client": "~1.17.2",
7477
"@temporalio/workflow": "~1.17.2",
78+
"js-yaml": "^4.1.1",
7579
"jsonwebtoken": "^9.0.0",
7680
"semver": "^7.6.0",
7781
"axios": "^1.16.1",
@@ -83,6 +87,7 @@
8387
"unzipper": "^0.12.3"
8488
},
8589
"devDependencies": {
90+
"@types/js-yaml": "^4.0.9",
8691
"@types/jsonwebtoken": "^9.0.0",
8792
"@types/node": "^20.8.2",
8893
"@types/semver": "^7.5.8",

‎services/apps/packages_worker/src/activities.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,3 +26,4 @@ export {
2626
} from './cargo/activities'
2727
export { enrichGoVersionsBatch, enrichGoStatusBatch } from './go/activities'
2828
export { processNuGetBatch } from './nuget/activities'
29+
export { processSecurityContactsBatch } from './security-contacts/activities'

‎services/apps/packages_worker/src/bin/security-contacts-worker.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
import { scheduleSecurityContactsIngestion } from '../security-contacts/schedule'
2+
import { svc } from '../service'
3+
4+
setImmediate(async () => {
5+
await svc.init()
6+
await scheduleSecurityContactsIngestion()
7+
await svc.start()
8+
})

‎services/apps/packages_worker/src/config.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,13 @@ export function getEnricherConfig() {
4646
}
4747
}
4848

49+
export function getSecurityContactsConfig() {
50+
return {
51+
// Sent on all registry calls; crates.io rejects requests without an identifying UA.
52+
userAgent: requireEnv('SECURITY_CONTACTS_USER_AGENT'),
53+
}
54+
}
55+
4956
export function getMavenConfig() {
5057
return {
5158
batchSize: requireEnvInt('MAVEN_FETCHER_BATCH_SIZE'),

0 commit comments

Comments
 (0)