Merge branch 'main' into feat/pypi-worker-with-downloads

Signed-off-by: anilb <epipav@gmail.com>

Author: epipav

backend/.env.dist.composed

@@ -35,3 +35,6 @@ CROWD_PACKAGES_DB_PORT=5432
 CROWD_PACKAGES_DB_USERNAME=postgres
 CROWD_PACKAGES_DB_PASSWORD=example
 CROWD_PACKAGES_DB_DATABASE=packages-db
+
+# security-contacts-worker
+SECURITY_CONTACTS_USER_AGENT="lfx-security-contacts-worker"

backend/.env.dist.local

@@ -185,6 +185,9 @@ ENRICHER_BATCH_SIZE=100
 ENRICHER_REPO_UPDATE_INTERVAL_HOURS=24
 ENRICHER_IDLE_SLEEP_SEC=60
 
+# security-contacts-worker
+SECURITY_CONTACTS_USER_AGENT="lfx-security-contacts-worker"
+
 OSSPCKGS_GCP_PROJECT=local-dev
 OSSPCKGS_GCS_BUCKET=local-dev
 OSSPCKGS_GCP_CREDENTIALS_B64=e30=

backend/src/osspckgs/migrations/V1782950400__security_contacts.sql

@@ -0,0 +1,27 @@
+CREATE TABLE IF NOT EXISTS security_contacts (
+    id              BIGSERIAL PRIMARY KEY,
+    repo_id         BIGINT NOT NULL REFERENCES repos(id) ON DELETE CASCADE,
+    channel         TEXT NOT NULL,
+    value           TEXT NOT NULL,
+    role            TEXT NOT NULL,
+    name            TEXT,
+    score           NUMERIC(4,3) NOT NULL,
+    confidence      TEXT NOT NULL,
+    provenance      JSONB NOT NULL DEFAULT '[]',
+    last_refreshed  TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    created_at      TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    updated_at      TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS security_contacts_repo_channel_value_uq
+    ON security_contacts (repo_id, channel, value);
+
+CREATE INDEX IF NOT EXISTS security_contacts_repo_confidence_idx
+    ON security_contacts (repo_id, confidence);
+
+ALTER TABLE repos ADD COLUMN IF NOT EXISTS pvr_enabled                 BOOL;
+ALTER TABLE repos ADD COLUMN IF NOT EXISTS security_policy_url         TEXT;
+ALTER TABLE repos ADD COLUMN IF NOT EXISTS vulnerability_reporting_url TEXT;
+ALTER TABLE repos ADD COLUMN IF NOT EXISTS bug_bounty_url              TEXT;
+ALTER TABLE repos ADD COLUMN IF NOT EXISTS security_txt_url            TEXT;
+ALTER TABLE repos ADD COLUMN IF NOT EXISTS contacts_last_refreshed     TIMESTAMPTZ;

backend/src/osspckgs/migrations/V1783036800__security_contacts_soft_delete.sql

@@ -0,0 +1,5 @@
+ALTER TABLE security_contacts ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ;
+
+CREATE INDEX IF NOT EXISTS security_contacts_repo_active_idx
+   ON security_contacts (repo_id)
+   WHERE deleted_at IS NULL;

pnpm-lock.yaml

@@ -1,4 +1,4 @@
 DOCKERFILE="./services/docker/Dockerfile.packages"
 CONTEXT="../"
 REPO="sjc.ocir.io/axbydjxa5zuh/packages"
-SERVICES="github-repos-enricher bq-dataset-ingest npm-worker pypi-worker maven-worker osv-worker dockerhub-sync cargo-worker go-worker nuget-worker"
+SERVICES="github-repos-enricher bq-dataset-ingest npm-worker pypi-worker maven-worker osv-worker dockerhub-sync cargo-worker go-worker nuget-worker security-contacts-worker"

scripts/builders/packages.env

@@ -0,0 +1,65 @@
+version: '3.1'
+
+x-env-args: &env-args
+  DOCKER_BUILDKIT: 1
+  NODE_ENV: docker
+  SERVICE: security-contacts-worker
+  SHELL: /bin/sh
+  SUPPRESS_NO_CONFIG_WARNING: 'true'
+  CROWD_TEMPORAL_TASKQUEUE: packages-worker
+
+services:
+  security-contacts-worker:
+    build:
+      context: ../../
+      dockerfile: ./scripts/services/docker/Dockerfile.packages
+    command: 'pnpm run start:security-contacts-worker'
+    working_dir: /usr/crowd/app/services/apps/packages_worker
+    env_file:
+      - ../../backend/.env.dist.local
+      - ../../backend/.env.dist.composed
+      - ../../backend/.env.override.local
+      - ../../backend/.env.override.composed
+    environment:
+      <<: *env-args
+    restart: always
+    networks:
+      - crowd-bridge
+
+  security-contacts-worker-dev:
+    build:
+      context: ../../
+      dockerfile: ./scripts/services/docker/Dockerfile.packages
+    command: 'pnpm run dev:security-contacts-worker'
+    working_dir: /usr/crowd/app/services/apps/packages_worker
+    env_file:
+      - ../../backend/.env.dist.local
+      - ../../backend/.env.dist.composed
+      - ../../backend/.env.override.local
+      - ../../backend/.env.override.composed
+    environment:
+      <<: *env-args
+    hostname: security-contacts-worker
+    networks:
+      - crowd-bridge
+    volumes:
+      - ../../services/libs/audit-logs/src:/usr/crowd/app/services/libs/audit-logs/src
+      - ../../services/libs/common/src:/usr/crowd/app/services/libs/common/src
+      - ../../services/libs/common_services/src:/usr/crowd/app/services/libs/common_services/src
+      - ../../services/libs/data-access-layer/src:/usr/crowd/app/services/libs/data-access-layer/src
+      - ../../services/libs/database/src:/usr/crowd/app/services/libs/database/src
+      - ../../services/libs/integrations/src:/usr/crowd/app/services/libs/integrations/src
+      - ../../services/libs/logging/src:/usr/crowd/app/services/libs/logging/src
+      - ../../services/libs/nango/src:/usr/crowd/app/services/libs/nango/src
+      - ../../services/libs/opensearch/src:/usr/crowd/app/services/libs/opensearch/src
+      - ../../services/libs/queue/src:/usr/crowd/app/services/libs/queue/src
+      - ../../services/libs/redis/src:/usr/crowd/app/services/libs/redis/src
+      - ../../services/libs/snowflake/src:/usr/crowd/app/services/libs/snowflake/src
+      - ../../services/libs/telemetry/src:/usr/crowd/app/services/libs/telemetry/src
+      - ../../services/libs/temporal/src:/usr/crowd/app/services/libs/temporal/src
+      - ../../services/libs/types/src:/usr/crowd/app/services/libs/types/src
+      - ../../services/apps/packages_worker/src:/usr/crowd/app/services/apps/packages_worker/src
+
+networks:
+  crowd-bridge:
+    external: true

scripts/services/security-contacts-worker.yaml

@@ -334,6 +334,7 @@ async def find_github_identity(github_username: str):
     WHERE
         platform = 'github'
         AND value = $1
+        AND "verified" = TRUE
         AND "deletedAt" is null
     LIMIT 1
     """
@@ -405,18 +406,45 @@ async def update_maintainer_run(repo_id: str, maintainer_file: str):
 
 
 async def get_maintainers_for_repo(repo_id: str):
+    # Active rows only (endDate IS NULL) — reappearing maintainers hit the "new"
+    # branch and get reactivated by upsert_maintainer's ON CONFLICT clause.
+    # verified=TRUE mirrors find_github_identity / find_maintainer_identity_by_email.
+    # platform/type are returned so the diff's safety guard can match identifiers
+    # by kind and avoid cross-platform value collisions (e.g. a GitHub username
+    # "foo" colliding with a same-named handle on another platform).
     maintainers_sql_query = """
-        SELECT mi.role, mi."originalRole", mi."repoUrl", mi."repoId", mi."identityId", mem.value as github_username
+        SELECT mi.role, mi."originalRole", mi."repoUrl", mi."repoId", mi."identityId",
+               mem.value as identity_value, mem.platform, mem.type
             FROM "maintainersInternal" mi
             JOIN "memberIdentities" mem ON mi."identityId" = mem.id
-        WHERE mi."repoId" = $1 AND mem.platform = 'github' AND mem.type = 'username' and mem.verified = True AND mem."deletedAt" is null
+        WHERE mi."repoId" = $1
+          AND mi."endDate" IS NULL
+          AND mem."verified" = TRUE
+          AND mem."deletedAt" is null
         """
     return await query(
         maintainers_sql_query,
         (repo_id,),
     )
 
 
+async def get_github_maintainer_usernames_for_repo(repo_id: str) -> set[str]:
+    """Return GitHub usernames of active maintainers for fork/parent-repo filtering."""
+    sql_query = """
+        SELECT mem.value
+            FROM "maintainersInternal" mi
+            JOIN "memberIdentities" mem ON mi."identityId" = mem.id
+        WHERE mi."repoId" = $1
+          AND mi."endDate" IS NULL
+          AND mem.platform = 'github'
+          AND mem.type = 'username'
+          AND mem."verified" = TRUE
+          AND mem."deletedAt" is null
+        """
+    rows = await query(sql_query, (repo_id,))
+    return {row["value"] for row in rows}
+
+
 async def set_maintainer_end_date(
     repo_id: str, identity_id: str, role: str, change_date: datetime
 ):