Skip to content

Commit 9a40daf

Browse files
themaroltthemarolt
authored
feat: pcc sync worker (CM-1086) (#4006)
Signed-off-by: Uroš Marolt <uros@marolt.me>
1 parent b6e7709 commit 9a40daf

35 files changed

Lines changed: 1696 additions & 62 deletions

‎backend/src/database/migrations/U1775312770__pcc-sync-worker-setup.sql‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
DROP INDEX IF EXISTS pcc_sync_errors_dedup_idx;
2+
3+
DROP TABLE IF EXISTS pcc_projects_sync_errors;
4+
5+
ALTER TABLE segments DROP COLUMN IF EXISTS maturity;

‎backend/src/database/migrations/V1775312770__pcc-sync-worker-setup.sql‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
-- Add maturity field to segments for PCC project_maturity_level sync
2+
ALTER TABLE segments ADD COLUMN IF NOT EXISTS maturity TEXT NULL;
3+
4+
-- Catch-all table for PCC sync issues that require manual review
5+
CREATE TABLE IF NOT EXISTS pcc_projects_sync_errors (
6+
id BIGSERIAL PRIMARY KEY,
7+
run_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
8+
external_project_id TEXT,
9+
external_project_slug TEXT,
10+
error_type TEXT NOT NULL,
11+
details JSONB,
12+
resolved BOOLEAN NOT NULL DEFAULT FALSE
13+
);
14+
15+
-- Deduplication index: one unresolved error per (project, error_type).
16+
-- On repeated daily exports the same error upserts in place instead of accumulating rows.
17+
-- Excludes rows where external_project_id IS NULL (e.g. SCHEMA_MISMATCH with no project id).
18+
CREATE UNIQUE INDEX IF NOT EXISTS pcc_sync_errors_dedup_idx
19+
ON pcc_projects_sync_errors (external_project_id, error_type)
20+
WHERE NOT resolved AND external_project_id IS NOT NULL;
21+
22+
-- Deduplication index for unidentifiable rows (no external_project_id).
23+
-- Keyed on (error_type, reason) so repeated daily exports don't accumulate duplicate rows
24+
-- for the same class of malformed input (e.g. rows missing PROJECT_ID/NAME/DEPTH).
25+
CREATE UNIQUE INDEX IF NOT EXISTS pcc_sync_errors_dedup_unknown_idx
26+
ON pcc_projects_sync_errors (error_type, (details->>'reason'))
27+
WHERE NOT resolved AND external_project_id IS NULL;

‎pnpm-lock.yaml‎

Lines changed: 68 additions & 18 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎scripts/services/docker/Dockerfile.pcc_sync_worker‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
FROM node:20-bullseye-slim AS builder
2+
3+
RUN apt-get update && apt-get install -y python3 make g++ && rm -rf /var/lib/apt/lists/*
4+
5+
WORKDIR /usr/crowd/app
6+
RUN npm install -g corepack@latest && corepack enable pnpm && corepack prepare pnpm@9.15.0 --activate
7+
8+
COPY ./pnpm-workspace.yaml ./pnpm-lock.yaml ./
9+
RUN pnpm fetch
10+
11+
COPY ./services ./services
12+
RUN pnpm i --frozen-lockfile
13+
14+
FROM node:20-bullseye-slim AS runner
15+
16+
RUN apt-get update && apt-get install -y ca-certificates && rm -rf /var/lib/apt/lists/*
17+
18+
WORKDIR /usr/crowd/app
19+
RUN npm install -g corepack@latest && corepack enable pnpm && corepack prepare pnpm@9.15.0 --activate
20+
21+
COPY --from=builder /usr/crowd/app/node_modules ./node_modules
22+
COPY --from=builder /usr/crowd/app/services/base.tsconfig.json ./services/base.tsconfig.json
23+
COPY --from=builder /usr/crowd/app/services/libs ./services/libs
24+
COPY --from=builder /usr/crowd/app/services/archetypes/ ./services/archetypes
25+
COPY --from=builder /usr/crowd/app/services/apps/pcc_sync_worker/ ./services/apps/pcc_sync_worker

‎scripts/services/docker/Dockerfile.pcc_sync_worker.dockerignore‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
**/.git
2+
**/node_modules
3+
**/venv*
4+
**/.webpack
5+
**/.serverless
6+
**/.env
7+
**/.env.*
8+
**/.idea
9+
**/.vscode
10+
**/dist
11+
.vscode/
12+
.github/
13+
frontend/
14+
scripts/
15+
.flake8
16+
*.md
17+
Makefile
18+
backend/

‎scripts/services/pcc-sync-worker.yaml‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
version: '3.1'
2+
3+
x-env-args: &env-args
4+
DOCKER_BUILDKIT: 1
5+
NODE_ENV: docker
6+
SERVICE: pcc-sync-worker
7+
CROWD_TEMPORAL_TASKQUEUE: pccSync
8+
SHELL: /bin/sh
9+
10+
services:
11+
pcc-sync-worker:
12+
build:
13+
context: ../../
14+
dockerfile: ./scripts/services/docker/Dockerfile.pcc_sync_worker
15+
command: 'pnpm run start'
16+
working_dir: /usr/crowd/app/services/apps/pcc_sync_worker
17+
env_file:
18+
- ../../backend/.env.dist.local
19+
- ../../backend/.env.dist.composed
20+
- ../../backend/.env.override.local
21+
- ../../backend/.env.override.composed
22+
environment:
23+
<<: *env-args
24+
restart: always
25+
networks:
26+
- crowd-bridge
27+
28+
pcc-sync-worker-dev:
29+
build:
30+
context: ../../
31+
dockerfile: ./scripts/services/docker/Dockerfile.pcc_sync_worker
32+
command: 'pnpm run dev'
33+
working_dir: /usr/crowd/app/services/apps/pcc_sync_worker
34+
env_file:
35+
- ../../backend/.env.dist.local
36+
- ../../backend/.env.dist.composed
37+
- ../../backend/.env.override.local
38+
- ../../backend/.env.override.composed
39+
environment:
40+
<<: *env-args
41+
hostname: pcc-sync-worker
42+
networks:
43+
- crowd-bridge
44+
volumes:
45+
- ../../services/libs/common/src:/usr/crowd/app/services/libs/common/src
46+
- ../../services/libs/logging/src:/usr/crowd/app/services/libs/logging/src
47+
- ../../services/libs/snowflake/src:/usr/crowd/app/services/libs/snowflake/src
48+
- ../../services/libs/temporal/src:/usr/crowd/app/services/libs/temporal/src
49+
- ../../services/apps/pcc_sync_worker/src:/usr/crowd/app/services/apps/pcc_sync_worker/src
50+
51+
networks:
52+
crowd-bridge:
53+
external: true

‎services/apps/pcc_sync_worker/package.json‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
{
2+
"name": "@crowd/pcc-sync-worker",
3+
"scripts": {
4+
"start": "CROWD_TEMPORAL_TASKQUEUE=pccSync SERVICE=pcc-sync-worker tsx src/index.ts",
5+
"start:debug": "CROWD_TEMPORAL_TASKQUEUE=pccSync SERVICE=pcc-sync-worker LOG_LEVEL=debug tsx src/index.ts",
6+
"start:debug:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=pccSync SERVICE=pcc-sync-worker LOG_LEVEL=debug tsx src/index.ts",
7+
"dev": "nodemon --watch src --watch ../../libs --ext ts --exec pnpm run start:debug",
8+
"dev:local": "nodemon --watch src --watch ../../libs --ext ts --exec pnpm run start:debug:local",
9+
"lint": "npx eslint --ext .ts src --max-warnings=0",
10+
"format": "npx prettier --write \"src/**/*.ts\"",
11+
"format-check": "npx prettier --check .",
12+
"tsc-check": "tsc --noEmit",
13+
"trigger-export": "SERVICE=pcc-sync-worker tsx src/scripts/triggerExport.ts",
14+
"trigger-export:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=pcc-sync-worker tsx src/scripts/triggerExport.ts",
15+
"trigger-cleanup": "SERVICE=pcc-sync-worker tsx src/scripts/triggerCleanup.ts",
16+
"trigger-cleanup:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=pcc-sync-worker tsx src/scripts/triggerCleanup.ts"
17+
},
18+
"dependencies": {
19+
"@crowd/archetype-standard": "workspace:*",
20+
"@crowd/archetype-worker": "workspace:*",
21+
"@crowd/common": "workspace:*",
22+
"@crowd/database": "workspace:*",
23+
"@crowd/types": "workspace:*",
24+
"@crowd/logging": "workspace:*",
25+
"@crowd/slack": "workspace:*",
26+
"@crowd/snowflake": "workspace:*",
27+
"@crowd/temporal": "workspace:*",
28+
"@temporalio/client": "~1.11.8",
29+
"@temporalio/workflow": "~1.11.8",
30+
"tsx": "^4.7.1",
31+
"typescript": "^5.6.3"
32+
},
33+
"devDependencies": {
34+
"nodemon": "^3.0.1"
35+
}
36+
}

0 commit comments

Comments
 (0)