Skip to content

Commit d36fc8a

Browse files
ulemonsulemons
committed
feat: adding project-discovery-worker
Signed-off-by: Umberto Sgueglia <usgueglia@contractor.linuxfoundation.org>
1 parent 4debebc commit d36fc8a

11 files changed

Lines changed: 322 additions & 0 deletions

File tree

‎pnpm-lock.yaml‎

Lines changed: 97 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎scripts/services/automatic-project-discovery-worker.yaml‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
version: '3.1'
2+
3+
x-env-args: &env-args
4+
DOCKER_BUILDKIT: 1
5+
NODE_ENV: docker
6+
SERVICE: automatic-project-discovery-worker
7+
CROWD_TEMPORAL_TASKQUEUE: automatic-project-discovery
8+
SHELL: /bin/sh
9+
10+
services:
11+
automatic-project-discovery-worker:
12+
build:
13+
context: ../../
14+
dockerfile: ./scripts/services/docker/Dockerfile.automatic_project_discovery_worker
15+
command: 'pnpm run start'
16+
working_dir: /usr/crowd/app/services/apps/automatic_project_discovery_worker
17+
env_file:
18+
- ../../backend/.env.dist.local
19+
- ../../backend/.env.dist.composed
20+
- ../../backend/.env.override.local
21+
- ../../backend/.env.override.composed
22+
environment:
23+
<<: *env-args
24+
restart: always
25+
networks:
26+
- crowd-bridge
27+
28+
automatic-project-discovery-worker-dev:
29+
build:
30+
context: ../../
31+
dockerfile: ./scripts/services/docker/Dockerfile.automatic_project_discovery_worker
32+
command: 'pnpm run dev'
33+
working_dir: /usr/crowd/app/services/apps/automatic_project_discovery_worker
34+
env_file:
35+
- ../../backend/.env.dist.local
36+
- ../../backend/.env.dist.composed
37+
- ../../backend/.env.override.local
38+
- ../../backend/.env.override.composed
39+
environment:
40+
<<: *env-args
41+
hostname: automatic-project-discovery-worker
42+
networks:
43+
- crowd-bridge
44+
volumes:
45+
- ../../services/libs/audit-logs/src:/usr/crowd/app/services/libs/audit-logs/src
46+
- ../../services/libs/common/src:/usr/crowd/app/services/libs/common/src
47+
- ../../services/libs/common_services/src:/usr/crowd/app/services/libs/common_services/src
48+
- ../../services/libs/data-access-layer/src:/usr/crowd/app/services/libs/data-access-layer/src
49+
- ../../services/libs/database/src:/usr/crowd/app/services/libs/database/src
50+
- ../../services/libs/integrations/src:/usr/crowd/app/services/libs/integrations/src
51+
- ../../services/libs/logging/src:/usr/crowd/app/services/libs/logging/src
52+
- ../../services/libs/nango/src:/usr/crowd/app/services/libs/nango/src
53+
- ../../services/libs/opensearch/src:/usr/crowd/app/services/libs/opensearch/src
54+
- ../../services/libs/queue/src:/usr/crowd/app/services/libs/queue/src
55+
- ../../services/libs/redis/src:/usr/crowd/app/services/libs/redis/src
56+
- ../../services/libs/snowflake/src:/usr/crowd/app/services/libs/snowflake/src
57+
- ../../services/libs/telemetry/src:/usr/crowd/app/services/libs/telemetry/src
58+
- ../../services/libs/temporal/src:/usr/crowd/app/services/libs/temporal/src
59+
- ../../services/libs/types/src:/usr/crowd/app/services/libs/types/src
60+
- ../../services/apps/automatic_project_discovery_worker/src:/usr/crowd/app/services/apps/automatic_project_discovery_worker/src
61+
62+
networks:
63+
crowd-bridge:
64+
external: true

‎scripts/services/docker/Dockerfile.automatic_project_discovery_worker‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
FROM node:20-alpine as builder
2+
3+
RUN apk add --no-cache python3 make g++
4+
5+
WORKDIR /usr/crowd/app
6+
RUN npm install -g corepack@latest && corepack enable pnpm && corepack prepare pnpm@9.15.0 --activate
7+
8+
COPY ./pnpm-workspace.yaml ./pnpm-lock.yaml ./
9+
RUN pnpm fetch
10+
11+
COPY ./services ./services
12+
RUN pnpm i --frozen-lockfile
13+
14+
FROM node:20-bookworm-slim as runner
15+
16+
WORKDIR /usr/crowd/app
17+
RUN npm install -g corepack@latest && corepack enable pnpm && corepack prepare pnpm@9.15.0 --activate && apt update && apt install -y ca-certificates --no-install-recommends && rm -rf /var/lib/apt/lists/*
18+
19+
COPY --from=builder /usr/crowd/app/node_modules ./node_modules
20+
COPY --from=builder /usr/crowd/app/services/base.tsconfig.json ./services/base.tsconfig.json
21+
COPY --from=builder /usr/crowd/app/services/libs ./services/libs
22+
COPY --from=builder /usr/crowd/app/services/archetypes/ ./services/archetypes
23+
COPY --from=builder /usr/crowd/app/services/apps/automatic_project_discovery_worker/ ./services/apps/automatic_project_discovery_worker

‎services/apps/automatic_project_discovery_worker/package.json‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
{
2+
"name": "@crowd/automatic-project-discovery-worker",
3+
"scripts": {
4+
"start": "CROWD_TEMPORAL_TASKQUEUE=automatic-project-discovery SERVICE=automatic-project-discovery-worker tsx src/main.ts",
5+
"start:debug:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=automatic-project-discovery SERVICE=automatic-project-discovery-worker LOG_LEVEL=trace tsx --inspect=0.0.0.0:9232 src/main.ts",
6+
"start:debug": "CROWD_TEMPORAL_TASKQUEUE=automatic-project-discovery SERVICE=automatic-project-discovery-worker LOG_LEVEL=trace tsx --inspect=0.0.0.0:9232 src/main.ts",
7+
"dev:local": "nodemon --watch src --watch ../../libs --ext ts --exec pnpm run start:debug:local",
8+
"dev": "nodemon --watch src --watch ../../libs --ext ts --exec pnpm run start:debug",
9+
"lint": "npx eslint --ext .ts src --max-warnings=0",
10+
"format": "npx prettier --write \"src/**/*.ts\"",
11+
"format-check": "npx prettier --check .",
12+
"tsc-check": "tsc --noEmit"
13+
},
14+
"dependencies": {
15+
"@crowd/archetype-standard": "workspace:*",
16+
"@crowd/archetype-worker": "workspace:*",
17+
"@crowd/common": "workspace:*",
18+
"@crowd/common_services": "workspace:*",
19+
"@crowd/data-access-layer": "workspace:*",
20+
"@crowd/logging": "workspace:*",
21+
"@crowd/redis": "workspace:*",
22+
"@crowd/temporal": "workspace:*",
23+
"@crowd/types": "workspace:*",
24+
"@temporalio/activity": "~1.11.8",
25+
"@temporalio/client": "~1.11.8",
26+
"@temporalio/workflow": "~1.11.8",
27+
"tsx": "^4.7.1",
28+
"typescript": "^5.6.3"
29+
},
30+
"devDependencies": {
31+
"@types/node": "^20.8.2",
32+
"nodemon": "^3.0.1"
33+
}
34+
}

‎services/apps/automatic_project_discovery_worker/src/activities.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
export * from './activities/activities'

‎services/apps/automatic_project_discovery_worker/src/activities/activities.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
import { getServiceLogger } from '@crowd/logging'
2+
3+
const log = getServiceLogger()
4+
5+
export async function logDiscoveryRun(): Promise<void> {
6+
log.info('Automatic project discovery workflow executed successfully.')
7+
}

‎services/apps/automatic_project_discovery_worker/src/main.ts‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
import { Config } from '@crowd/archetype-standard'
2+
import { Options, ServiceWorker } from '@crowd/archetype-worker'
3+
4+
import { scheduleProjectDiscovery } from './schedules/scheduleProjectDiscovery'
5+
6+
const config: Config = {
7+
envvars: [],
8+
producer: {
9+
enabled: false,
10+
},
11+
temporal: {
12+
enabled: true,
13+
},
14+
redis: {
15+
enabled: false,
16+
},
17+
}
18+
19+
const options: Options = {
20+
postgres: {
21+
enabled: false,
22+
},
23+
opensearch: {
24+
enabled: false,
25+
},
26+
}
27+
28+
export const svc = new ServiceWorker(config, options)
29+
30+
setImmediate(async () => {
31+
await svc.init()
32+
33+
await scheduleProjectDiscovery()
34+
35+
await svc.start()
36+
})

‎services/apps/automatic_project_discovery_worker/src/schedules/scheduleProjectDiscovery.ts‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
import { ScheduleAlreadyRunning, ScheduleOverlapPolicy } from '@temporalio/client'
2+
3+
import { svc } from '../main'
4+
import { discoverProjects } from '../workflows'
5+
6+
const DEFAULT_CRON = '0 2 * * *' // Daily at 2:00 AM
7+
8+
export const scheduleProjectDiscovery = async () => {
9+
const cronExpression = process.env.CROWD_AUTOMATIC_PROJECT_DISCOVERY_CRON || DEFAULT_CRON
10+
11+
svc.log.info(`Scheduling project discovery with cron: ${cronExpression}`)
12+
13+
try {
14+
await svc.temporal.schedule.create({
15+
scheduleId: 'automaticProjectDiscovery',
16+
spec: {
17+
cronExpressions: [cronExpression],
18+
},
19+
policies: {
20+
overlap: ScheduleOverlapPolicy.SKIP,
21+
catchupWindow: '1 minute',
22+
},
23+
action: {
24+
type: 'startWorkflow',
25+
workflowType: discoverProjects,
26+
taskQueue: 'automatic-project-discovery',
27+
retry: {
28+
initialInterval: '15 seconds',
29+
backoffCoefficient: 2,
30+
maximumAttempts: 3,
31+
},
32+
},
33+
})
34+
} catch (err) {
35+
if (err instanceof ScheduleAlreadyRunning) {
36+
svc.log.info('Schedule already registered in Temporal.')
37+
svc.log.info('Configuration may have changed since. Please make sure they are in sync.')
38+
} else {
39+
throw new Error(err)
40+
}
41+
}
42+
}

‎services/apps/automatic_project_discovery_worker/src/workflows.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
import { discoverProjects } from './workflows/discoverProjects'
2+
3+
export { discoverProjects }

‎services/apps/automatic_project_discovery_worker/src/workflows/discoverProjects.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
import { proxyActivities } from '@temporalio/workflow'
2+
3+
import type * as activities from '../activities'
4+
5+
const activity = proxyActivities<typeof activities>({
6+
startToCloseTimeout: '1 minutes',
7+
})
8+
9+
export async function discoverProjects(): Promise<void> {
10+
await activity.logDiscoveryRun()
11+
}

0 commit comments

Comments
 (0)