From 81f23a4bc4cdde4550988abe15058852fbb4e4d4 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Wed, 10 Jun 2026 16:11:57 +0200 Subject: [PATCH 1/9] feat: adding stewardship tables and small fixis Signed-off-by: Umberto Sgueglia --- .../public/v1/packages/batchGetStewardship.ts | 22 +--- .../src/api/public/v1/packages/mockData.ts | 32 +++-- .../src/api/public/v1/packages/openapi.yaml | 25 +++- backend/src/api/public/v1/packages/types.ts | 40 ++++++ .../V1781094067__stewardship-tables.sql | 123 ++++++++++++++++++ 5 files changed, 211 insertions(+), 31 deletions(-) create mode 100644 backend/src/api/public/v1/packages/types.ts create mode 100644 backend/src/osspckgs/migrations/V1781094067__stewardship-tables.sql diff --git a/backend/src/api/public/v1/packages/batchGetStewardship.ts b/backend/src/api/public/v1/packages/batchGetStewardship.ts index 9c140a8eca..529eaa2f8c 100644 --- a/backend/src/api/public/v1/packages/batchGetStewardship.ts +++ b/backend/src/api/public/v1/packages/batchGetStewardship.ts @@ -5,6 +5,7 @@ import { ok } from '@/utils/api' import { validateOrThrow } from '@/utils/validation' import { MOCK_DETAILS } from './mockData' +import type { OpenVulns, StewardshipSummary } from './types' const MAX_PURLS = 100 @@ -15,19 +16,6 @@ const bodySchema = z.object({ .max(MAX_PURLS, `Maximum ${MAX_PURLS} purls per request`), }) -interface StewardshipSummary { - name: string - ecosystem: string - lifecycle: string - health: number - impact: number - openVulns: { low: number; medium: number; high: number; critical: number } - stewardship: string - stewards: null - lastActivityAt: null - lastActivityDescription: null -} - // TODO: replace with real DB queries once stewardship tables land export async function batchGetStewardship(req: Request, res: Response): Promise { const { purls } = validateOrThrow(bodySchema, req.body) @@ -38,11 +26,9 @@ export async function batchGetStewardship(req: Request, res: Response): Promise< if (!detail) { packages[purl] = null } else { - const openVulns = { low: 0, medium: 0, high: 0, critical: 0 } + const openVulns: OpenVulns = { low: 0, medium: 0, high: 0, critical: 0 } for (const advisory of detail.security.advisories) { - if (advisory.severity in openVulns) { - openVulns[advisory.severity] += 1 - } + openVulns[advisory.severity] += 1 } packages[purl] = { name: detail.name, @@ -51,7 +37,7 @@ export async function batchGetStewardship(req: Request, res: Response): Promise< health: detail.general.healthScore.total, impact: detail.general.impact.impactScore, openVulns, - stewardship: 'unassigned', + stewardship: detail.stewardship.status, stewards: null, lastActivityAt: null, lastActivityDescription: null, diff --git a/backend/src/api/public/v1/packages/mockData.ts b/backend/src/api/public/v1/packages/mockData.ts index 7f68414fd3..279d509b68 100644 --- a/backend/src/api/public/v1/packages/mockData.ts +++ b/backend/src/api/public/v1/packages/mockData.ts @@ -1,14 +1,16 @@ +import type { Lifecycle, OpenVulns, SeverityLevel, Steward, StewardshipStatus } from './types' + export interface MockPackageListItem { purl: string name: string ecosystem: string health: number impact: number - lifecycle: 'active' | 'stable' | 'declining' | 'abandoned' + lifecycle: Lifecycle maintainerBusFactor: number - openVulns: { low: number; medium: number; high: number; critical: number } - stewardship: string - steward: null + openVulns: OpenVulns + stewardship: StewardshipStatus + stewards: Steward | null } export interface MockPackageDetail { @@ -30,19 +32,19 @@ export interface MockPackageDetail { transitiveReach: string } riskSignals: { - lifecycle: string + lifecycle: Lifecycle maintainerBusFactor: number lastRelease: string hasSecurityFile: null openSSFScorecard: number } } - assessment: Record + assessment: Record security: { securityContacts: null advisories: Array<{ osvId: string - severity: 'critical' | 'high' | 'medium' | 'low' + severity: SeverityLevel resolution: null }> cvd: { @@ -56,7 +58,12 @@ export interface MockPackageDetail { repositoryMapping: { declaredRepo: string; mappingConfidence: number; lastCommitAt: string } supplyChainIntegrity: { buildProvenance: null; signedReleases: null } } - history: Record + stewardship: { + status: StewardshipStatus + stewards: Steward | null + lastActivityAt: string | null + } + history: Record } export const MOCK_PACKAGES: MockPackageListItem[] = [ @@ -70,7 +77,7 @@ export const MOCK_PACKAGES: MockPackageListItem[] = [ maintainerBusFactor: 1, openVulns: { low: 0, medium: 0, high: 1, critical: 0 }, stewardship: 'unassigned', - steward: null, + stewards: null, }, { purl: 'pkg:maven/org.apache.commons/commons-lang3@3.12.0', @@ -82,7 +89,7 @@ export const MOCK_PACKAGES: MockPackageListItem[] = [ maintainerBusFactor: 3, openVulns: { low: 0, medium: 0, high: 0, critical: 0 }, stewardship: 'unassigned', - steward: null, + stewards: null, }, { purl: 'pkg:npm/minimist@1.2.6', @@ -94,7 +101,7 @@ export const MOCK_PACKAGES: MockPackageListItem[] = [ maintainerBusFactor: 1, openVulns: { low: 0, medium: 1, high: 0, critical: 1 }, stewardship: 'unassigned', - steward: null, + stewards: null, }, ] @@ -144,6 +151,7 @@ export const MOCK_DETAILS: Record = { }, supplyChainIntegrity: { buildProvenance: null, signedReleases: null }, }, + stewardship: { status: 'unassigned', stewards: null, lastActivityAt: null }, history: {}, }, 'pkg:maven/org.apache.commons/commons-lang3@3.12.0': { @@ -191,6 +199,7 @@ export const MOCK_DETAILS: Record = { }, supplyChainIntegrity: { buildProvenance: null, signedReleases: null }, }, + stewardship: { status: 'unassigned', stewards: null, lastActivityAt: null }, history: {}, }, 'pkg:npm/minimist@1.2.6': { @@ -241,6 +250,7 @@ export const MOCK_DETAILS: Record = { }, supplyChainIntegrity: { buildProvenance: null, signedReleases: null }, }, + stewardship: { status: 'unassigned', stewards: null, lastActivityAt: null }, history: {}, }, } diff --git a/backend/src/api/public/v1/packages/openapi.yaml b/backend/src/api/public/v1/packages/openapi.yaml index 5f5d2936f6..e731059abe 100644 --- a/backend/src/api/public/v1/packages/openapi.yaml +++ b/backend/src/api/public/v1/packages/openapi.yaml @@ -105,7 +105,7 @@ components: StewardshipSummary: type: object description: Null if the purl is not found in CDP. - required: [name, ecosystem, stewardship, stewards, lastActivityAt, lastActivityDescription] + required: [name, ecosystem, stewardship, stewards, openVulns, lastActivityAt, lastActivityDescription] properties: name: type: string @@ -193,7 +193,7 @@ components: - type: 'null' stewardship: $ref: '#/components/schemas/StewardshipStatus' - steward: + stewards: description: Single assigned steward or null. oneOf: - $ref: '#/components/schemas/Steward' @@ -411,6 +411,23 @@ components: type: - string - 'null' + stewardship: + type: object + description: Stewardship state. In v1 always unassigned with no stewards or activity. + properties: + status: + $ref: '#/components/schemas/StewardshipStatus' + stewards: + description: Single assigned steward or null. Null in v1. + oneOf: + - $ref: '#/components/schemas/Steward' + - type: 'null' + lastActivityAt: + type: + - string + - 'null' + format: date-time + description: Null in v1. history: type: object description: Package history data. Empty in v1. @@ -657,6 +674,10 @@ paths: supplyChainIntegrity: buildProvenance: null signedReleases: null + stewardship: + status: unassigned + stewards: null + lastActivityAt: null history: {} '404': description: Package not found. diff --git a/backend/src/api/public/v1/packages/types.ts b/backend/src/api/public/v1/packages/types.ts new file mode 100644 index 0000000000..29a2c27df8 --- /dev/null +++ b/backend/src/api/public/v1/packages/types.ts @@ -0,0 +1,40 @@ +export type StewardshipStatus = + | 'unassigned' + | 'open' + | 'assessing' + | 'active' + | 'needs_attention' + | 'escalated' + | 'blocked' + | 'inactive' + +export type Lifecycle = 'active' | 'stable' | 'declining' | 'abandoned' + +export type SeverityLevel = 'critical' | 'high' | 'medium' | 'low' + +export interface OpenVulns { + low: number + medium: number + high: number + critical: number +} + +export interface Steward { + userId: string + name: string + role: 'lead' | 'co_steward' + assignedAt: string +} + +export interface StewardshipSummary { + name: string + ecosystem: string + lifecycle: Lifecycle | null + health: number | null + impact: number | null + openVulns: OpenVulns | null + stewardship: StewardshipStatus + stewards: Steward | null + lastActivityAt: string | null + lastActivityDescription: string | null +} diff --git a/backend/src/osspckgs/migrations/V1781094067__stewardship-tables.sql b/backend/src/osspckgs/migrations/V1781094067__stewardship-tables.sql new file mode 100644 index 0000000000..4236dc172a --- /dev/null +++ b/backend/src/osspckgs/migrations/V1781094067__stewardship-tables.sql @@ -0,0 +1,123 @@ +-- Stewardship tables for the OSSPREY Self Serve program (v1). +-- In v1: only `stewardships` is populated (one unassigned row per critical package). +-- All other tables are schema-only — empty until v2 write flows land. + +CREATE TABLE IF NOT EXISTS stewardships ( + id BIGSERIAL PRIMARY KEY, + package_id BIGINT NOT NULL REFERENCES packages(id), + status TEXT NOT NULL, -- 'unassigned'|'open'|'assessing'|'active'|'needs_attention'|'escalated'|'blocked'|'inactive' + origin TEXT NOT NULL, -- 'auto_imported'|'self_claimed'|'assigned'|'opened_for_claim' + version INT NOT NULL DEFAULT 1, + opened_at TIMESTAMPTZ, + last_status_at TIMESTAMPTZ, + inactive_reason TEXT, -- 'quarterly_cadence_missed'|'stepped_down'|'no_longer_critical' + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (package_id) +); + +CREATE INDEX IF NOT EXISTS stewardships_status_idx + ON stewardships (status); +CREATE INDEX IF NOT EXISTS stewardships_last_status_at_active_idx + ON stewardships (last_status_at) WHERE status = 'active'; + +-- Many-to-many stewards. Empty in v1; soft-delete preserves historical membership. +CREATE TABLE IF NOT EXISTS stewardship_stewards ( + id BIGSERIAL PRIMARY KEY, + stewardship_id BIGINT NOT NULL REFERENCES stewardships(id), + user_id TEXT NOT NULL, + role TEXT NOT NULL, -- 'lead'|'co_steward' + assigned_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + assigned_by TEXT, + deleted_at TIMESTAMPTZ +); + +CREATE UNIQUE INDEX IF NOT EXISTS stewardship_stewards_active_unique + ON stewardship_stewards (stewardship_id, user_id) + WHERE deleted_at IS NULL; +CREATE INDEX IF NOT EXISTS stewardship_stewards_user_id_active_idx + ON stewardship_stewards (user_id) WHERE deleted_at IS NULL; + +-- Append-only audit log. Empty in v1. +CREATE TABLE IF NOT EXISTS stewardship_activity ( + id BIGSERIAL PRIMARY KEY, + stewardship_id BIGINT NOT NULL REFERENCES stewardships(id), + actor_user_id TEXT, -- NULL for system events + actor_type TEXT NOT NULL, -- 'user'|'system' + activity_type TEXT NOT NULL, -- 'state_changed'|'assessment_completed'|'assessment_flagged'| + -- 'remediation_logged'|'status_update'|'escalation'| + -- 'escalation_resolved'|'blocker_added'|'blocker_resolved'| + -- 'steward_added'|'steward_removed' + content TEXT, + metadata JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS stewardship_activity_stewardship_id_created_at_idx + ON stewardship_activity (stewardship_id, created_at DESC); + +-- One current assessment per stewardship; historical ones preserved via superseded_at. +CREATE TABLE IF NOT EXISTS stewardship_assessments ( + id BIGSERIAL PRIMARY KEY, + stewardship_id BIGINT NOT NULL REFERENCES stewardships(id), + posture TEXT, + summary TEXT, + security_contact TEXT, + disclosure_preference TEXT, + tier_0_ready BOOL NOT NULL DEFAULT FALSE, + monitoring_plan TEXT, + draft BOOL NOT NULL DEFAULT TRUE, + completed_at TIMESTAMPTZ, + completed_by TEXT, + reviewed BOOL NOT NULL DEFAULT FALSE, + reviewed_at TIMESTAMPTZ, + reviewed_by TEXT, + flagged BOOL NOT NULL DEFAULT FALSE, + flag_note TEXT, + superseded_at TIMESTAMPTZ, + superseded_by_id BIGINT REFERENCES stewardship_assessments(id), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -- tracks mutations: reviewed, flagged, superseded_at +); + +CREATE INDEX IF NOT EXISTS stewardship_assessments_stewardship_id_superseded_at_idx + ON stewardship_assessments (stewardship_id, superseded_at); +CREATE UNIQUE INDEX IF NOT EXISTS stewardship_assessments_one_current + ON stewardship_assessments (stewardship_id) + WHERE superseded_at IS NULL; + +-- Per-dimension findings. assessment_id links a finding to the assessment that produced it. +CREATE TABLE IF NOT EXISTS stewardship_findings ( + id BIGSERIAL PRIMARY KEY, + stewardship_id BIGINT NOT NULL REFERENCES stewardships(id), + assessment_id BIGINT REFERENCES stewardship_assessments(id), -- NULL until assessment flow lands in v2 + dimension TEXT NOT NULL, -- 'maintainer_health'|'security_posture'|'vulnerability_exposure'| + -- 'dependency_risk'|'supply_chain_integrity'|'release_health' + severity TEXT NOT NULL, -- 'critical'|'high'|'medium'|'low'|'informational' + finding TEXT NOT NULL, + evidence TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS stewardship_findings_stewardship_id_idx + ON stewardship_findings (stewardship_id); +CREATE INDEX IF NOT EXISTS stewardship_findings_dimension_severity_idx + ON stewardship_findings (dimension, severity); + +-- Concrete remediation actions. Empty in v1. +CREATE TABLE IF NOT EXISTS stewardship_remediation_actions ( + id BIGSERIAL PRIMARY KEY, + stewardship_id BIGINT NOT NULL REFERENCES stewardships(id), + finding_id BIGINT REFERENCES stewardship_findings(id), + action TEXT NOT NULL, + status TEXT NOT NULL, -- 'pending'|'in_progress'|'done'|'blocked'|'abandoned' + url TEXT, + notes TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + completed_at TIMESTAMPTZ +); + +CREATE INDEX IF NOT EXISTS stewardship_remediation_actions_stewardship_id_status_idx + ON stewardship_remediation_actions (stewardship_id, status); From b7f1724acb08d4d47c4110520acbd964fe4df289 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Wed, 10 Jun 2026 16:13:50 +0200 Subject: [PATCH 2/9] fix: openapi refactor Signed-off-by: Umberto Sgueglia --- backend/src/api/public/v1/packages/openapi.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/src/api/public/v1/packages/openapi.yaml b/backend/src/api/public/v1/packages/openapi.yaml index e731059abe..bcb1c1bb74 100644 --- a/backend/src/api/public/v1/packages/openapi.yaml +++ b/backend/src/api/public/v1/packages/openapi.yaml @@ -105,7 +105,8 @@ components: StewardshipSummary: type: object description: Null if the purl is not found in CDP. - required: [name, ecosystem, stewardship, stewards, openVulns, lastActivityAt, lastActivityDescription] + required: + [name, ecosystem, stewardship, stewards, openVulns, lastActivityAt, lastActivityDescription] properties: name: type: string From f1421c2b9ac87933f1d7b1b28582117b7ada1494 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Thu, 11 Jun 2026 10:30:51 +0200 Subject: [PATCH 3/9] fix: align stewards schema to array type and fix unstewardedOnly null filter Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Umberto Sgueglia --- .../public/v1/packages/batchGetStewardship.ts | 2 +- .../src/api/public/v1/packages/listPackages.ts | 2 +- backend/src/api/public/v1/packages/mockData.ts | 6 +++--- .../src/api/public/v1/packages/openapi.yaml | 18 ++++++++++++------ backend/src/api/public/v1/packages/types.ts | 4 ++-- 5 files changed, 19 insertions(+), 13 deletions(-) diff --git a/backend/src/api/public/v1/packages/batchGetStewardship.ts b/backend/src/api/public/v1/packages/batchGetStewardship.ts index 529eaa2f8c..e888f1b5c8 100644 --- a/backend/src/api/public/v1/packages/batchGetStewardship.ts +++ b/backend/src/api/public/v1/packages/batchGetStewardship.ts @@ -38,7 +38,7 @@ export async function batchGetStewardship(req: Request, res: Response): Promise< impact: detail.general.impact.impactScore, openVulns, stewardship: detail.stewardship.status, - stewards: null, + stewards: detail.stewardship.stewards, lastActivityAt: null, lastActivityDescription: null, } diff --git a/backend/src/api/public/v1/packages/listPackages.ts b/backend/src/api/public/v1/packages/listPackages.ts index 2077cb0d09..6e912c3581 100644 --- a/backend/src/api/public/v1/packages/listPackages.ts +++ b/backend/src/api/public/v1/packages/listPackages.ts @@ -47,7 +47,7 @@ export async function listPackages(req: Request, res: Response): Promise { if (ecosystem && p.ecosystem !== ecosystem) return false if (lifecycle && p.lifecycle !== lifecycle) return false if (busFactor1Only && p.maintainerBusFactor !== 1) return false - if (unstewardedOnly && p.stewardship !== 'unassigned') return false + if (unstewardedOnly && p.stewardship !== null && p.stewardship !== 'unassigned') return false if (staleOnly) { const lastRelease = MOCK_DETAILS[p.purl]?.general.riskSignals.lastRelease if (!lastRelease || new Date(lastRelease) >= staleThreshold) return false diff --git a/backend/src/api/public/v1/packages/mockData.ts b/backend/src/api/public/v1/packages/mockData.ts index 279d509b68..3b78a2b5ae 100644 --- a/backend/src/api/public/v1/packages/mockData.ts +++ b/backend/src/api/public/v1/packages/mockData.ts @@ -9,8 +9,8 @@ export interface MockPackageListItem { lifecycle: Lifecycle maintainerBusFactor: number openVulns: OpenVulns - stewardship: StewardshipStatus - stewards: Steward | null + stewardship: StewardshipStatus | null + stewards: Steward[] | null } export interface MockPackageDetail { @@ -60,7 +60,7 @@ export interface MockPackageDetail { } stewardship: { status: StewardshipStatus - stewards: Steward | null + stewards: Steward[] | null lastActivityAt: string | null } history: Record diff --git a/backend/src/api/public/v1/packages/openapi.yaml b/backend/src/api/public/v1/packages/openapi.yaml index bcb1c1bb74..17bf9d3aab 100644 --- a/backend/src/api/public/v1/packages/openapi.yaml +++ b/backend/src/api/public/v1/packages/openapi.yaml @@ -136,9 +136,11 @@ components: stewardship: $ref: '#/components/schemas/StewardshipStatus' stewards: - description: Single steward or null. Empty in v1. + description: Assigned stewards or null. Empty in v1. oneOf: - - $ref: '#/components/schemas/Steward' + - type: array + items: + $ref: '#/components/schemas/Steward' - type: 'null' lastActivityAt: type: @@ -195,9 +197,11 @@ components: stewardship: $ref: '#/components/schemas/StewardshipStatus' stewards: - description: Single assigned steward or null. + description: Assigned stewards or null. oneOf: - - $ref: '#/components/schemas/Steward' + - type: array + items: + $ref: '#/components/schemas/Steward' - type: 'null' # ── Package detail ─────────────────────────────────────────────────────────── @@ -419,9 +423,11 @@ components: status: $ref: '#/components/schemas/StewardshipStatus' stewards: - description: Single assigned steward or null. Null in v1. + description: Assigned stewards or null. Null in v1. oneOf: - - $ref: '#/components/schemas/Steward' + - type: array + items: + $ref: '#/components/schemas/Steward' - type: 'null' lastActivityAt: type: diff --git a/backend/src/api/public/v1/packages/types.ts b/backend/src/api/public/v1/packages/types.ts index 29a2c27df8..7fad33aed5 100644 --- a/backend/src/api/public/v1/packages/types.ts +++ b/backend/src/api/public/v1/packages/types.ts @@ -33,8 +33,8 @@ export interface StewardshipSummary { health: number | null impact: number | null openVulns: OpenVulns | null - stewardship: StewardshipStatus - stewards: Steward | null + stewardship: StewardshipStatus | null + stewards: Steward[] | null lastActivityAt: string | null lastActivityDescription: string | null } From a739c9f578470c6873f3d348088f265c2efeaeae Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Wed, 10 Jun 2026 18:10:21 +0200 Subject: [PATCH 4/9] feat: add backfill script Signed-off-by: Umberto Sgueglia --- services/apps/packages_worker/package.json | 2 + .../src/bin/stewardship-backfill.ts | 45 ++++++++++++++ .../src/stewardship/runStewardshipBackfill.ts | 60 +++++++++++++++++++ services/libs/data-access-layer/src/index.ts | 1 + .../data-access-layer/src/osspckgs/index.ts | 1 + .../src/osspckgs/stewardships.ts | 59 ++++++++++++++++++ 6 files changed, 168 insertions(+) create mode 100644 services/apps/packages_worker/src/bin/stewardship-backfill.ts create mode 100644 services/apps/packages_worker/src/stewardship/runStewardshipBackfill.ts create mode 100644 services/libs/data-access-layer/src/osspckgs/stewardships.ts diff --git a/services/apps/packages_worker/package.json b/services/apps/packages_worker/package.json index c513f8b10b..3f5f9789b4 100644 --- a/services/apps/packages_worker/package.json +++ b/services/apps/packages_worker/package.json @@ -13,6 +13,8 @@ "dev:pagerank": "tsx --expose-gc src/criticality/run-pagerank.ts", "start:pom-fetcher": "SERVICE=pom-fetcher tsx src/bin/pom-fetcher.ts", "backfill:maven": "SERVICE=maven tsx src/bin/maven-backfill.ts", + "backfill:stewardship": "SERVICE=stewardship-backfill tsx src/bin/stewardship-backfill.ts", + "backfill:stewardship:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=stewardship-backfill LOG_LEVEL=info tsx src/bin/stewardship-backfill.ts", "dev:packages-worker": "CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=packages-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9233 src/bin/packages-worker.ts", "dev:criticality-worker": "CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=criticality-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9237 src/bin/criticality-worker.ts", "start:maven-worker": "CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=maven-worker tsx src/bin/maven-worker.ts", diff --git a/services/apps/packages_worker/src/bin/stewardship-backfill.ts b/services/apps/packages_worker/src/bin/stewardship-backfill.ts new file mode 100644 index 0000000000..6cabbd640f --- /dev/null +++ b/services/apps/packages_worker/src/bin/stewardship-backfill.ts @@ -0,0 +1,45 @@ +import { getServiceLogger } from '@crowd/logging' + +import { getPackagesDb } from '../db' +import { runStewardshipBackfill } from '../stewardship/runStewardshipBackfill' + +const log = getServiceLogger() + +let shuttingDown = false + +// Graceful stop: finish the in-flight batch, then exit. Safe to interrupt — every +// write is ON CONFLICT DO NOTHING so re-running resumes where it left off. +const shutdown = () => { + if (shuttingDown) return + shuttingDown = true + log.info('Shutting down stewardship backfill (stopping after the current batch)...') +} + +process.on('SIGINT', shutdown) +process.on('SIGTERM', shutdown) + +const main = async () => { + log.info('stewardship backfill starting...') + + const qx = await getPackagesDb() + await qx.selectOne('SELECT 1') + log.info('Connected to packages-db.') + + const rawBatchSize = parseInt(process.env.STEWARDSHIP_BACKFILL_BATCH_SIZE ?? '10000', 10) + if (!Number.isFinite(rawBatchSize) || rawBatchSize <= 0) { + throw new Error( + `STEWARDSHIP_BACKFILL_BATCH_SIZE must be a positive integer, got: ${process.env.STEWARDSHIP_BACKFILL_BATCH_SIZE}`, + ) + } + const batchSize = rawBatchSize + + const totals = await runStewardshipBackfill(qx, { batchSize }, () => shuttingDown) + + log.info({ ...totals }, 'stewardship backfill complete') + process.exit(0) +} + +main().catch((err) => { + log.error({ err }, 'stewardship backfill fatal error') + process.exit(1) +}) diff --git a/services/apps/packages_worker/src/stewardship/runStewardshipBackfill.ts b/services/apps/packages_worker/src/stewardship/runStewardshipBackfill.ts new file mode 100644 index 0000000000..526f6f9f74 --- /dev/null +++ b/services/apps/packages_worker/src/stewardship/runStewardshipBackfill.ts @@ -0,0 +1,60 @@ +import { + QueryExecutor, + insertUnassignedStewardships, + listCriticalPackagesWithoutStewardship, +} from '@crowd/data-access-layer' +import { getServiceChildLogger } from '@crowd/logging' + +const log = getServiceChildLogger('stewardship-backfill') + +export interface BackfillResult { + inserted: number + skipped: number + batches: number +} + +interface BackfillOptions { + batchSize: number +} + +/** + * Seeds one `stewardships` row (status=unassigned, origin=auto_imported) for + * every critical package that doesn't already have one. Idempotent: ON CONFLICT + * DO NOTHING means re-running is safe and will just report 0 inserts. + * + * Designed to be called from a Temporal activity or directly from the bin script. + * The `isStopping` callback lets the caller signal a graceful shutdown between + * batches — the function returns the totals collected so far. + */ +export async function runStewardshipBackfill( + qx: QueryExecutor, + options: BackfillOptions, + isStopping: () => boolean = () => false, +): Promise { + const { batchSize } = options + let lastId = 0 + let inserted = 0 + let skipped = 0 + let batches = 0 + + while (!isStopping()) { + const ids = await listCriticalPackagesWithoutStewardship(qx, { + afterId: lastId, + limit: batchSize, + }) + + if (ids.length === 0) break + + const batchInserted = await insertUnassignedStewardships(qx, ids) + const batchSkipped = ids.length - batchInserted + + inserted += batchInserted + skipped += batchSkipped + batches++ + lastId = ids[ids.length - 1] + + log.info({ batches, inserted, skipped, lastId, batchInserted, batchSkipped }, 'Batch complete.') + } + + return { inserted, skipped, batches } +} diff --git a/services/libs/data-access-layer/src/index.ts b/services/libs/data-access-layer/src/index.ts index ab7c60db59..1070d84c1c 100644 --- a/services/libs/data-access-layer/src/index.ts +++ b/services/libs/data-access-layer/src/index.ts @@ -20,4 +20,5 @@ export * from './osspckgs/ingestJobs' export * from './osspckgs/maintainers' export * from './osspckgs/packages' export * from './osspckgs/repos' +export * from './osspckgs/stewardships' export * from './osspckgs/versions' diff --git a/services/libs/data-access-layer/src/osspckgs/index.ts b/services/libs/data-access-layer/src/osspckgs/index.ts index d235aa9713..65228ac4d7 100644 --- a/services/libs/data-access-layer/src/osspckgs/index.ts +++ b/services/libs/data-access-layer/src/osspckgs/index.ts @@ -3,3 +3,4 @@ export * from './packages' export * from './maintainers' export * from './versions' export * from './repos' +export * from './stewardships' diff --git a/services/libs/data-access-layer/src/osspckgs/stewardships.ts b/services/libs/data-access-layer/src/osspckgs/stewardships.ts new file mode 100644 index 0000000000..17579147e7 --- /dev/null +++ b/services/libs/data-access-layer/src/osspckgs/stewardships.ts @@ -0,0 +1,59 @@ +import { QueryExecutor } from '../queryExecutor' + +/** + * Returns a page of critical package ids that do not yet have a stewardship row, + * ordered by id ascending. Used as the cursor-based pagination source for the + * stewardship backfill. + */ +export async function listCriticalPackagesWithoutStewardship( + qx: QueryExecutor, + options: { afterId: number; limit: number }, +): Promise { + // pg returns BIGINT columns as strings; Number() is safe here because + // package ids are well within JS safe-integer range. + const rows: Array<{ id: string | number }> = await qx.select( + ` + SELECT p.id + FROM packages p + LEFT JOIN stewardships s ON s.package_id = p.id + WHERE p.is_critical = true + AND p.id > $(afterId) + AND s.package_id IS NULL + ORDER BY p.id ASC + LIMIT $(limit) + `, + options, + ) + return rows.map((r) => Number(r.id)) +} + +/** + * Inserts one unassigned stewardship row per package id. Idempotent: + * ON CONFLICT DO NOTHING skips ids that already have a row. + * Returns the number of rows actually inserted. + * + * Re-checks is_critical at insert time to guard against concurrent criticality + * changes between the SELECT and INSERT phases. + */ +export async function insertUnassignedStewardships( + qx: QueryExecutor, + packageIds: number[], +): Promise { + if (packageIds.length === 0) return 0 + const result: { count: string } = await qx.selectOne( + ` + WITH ins AS ( + INSERT INTO stewardships (package_id, status, origin, opened_at, last_status_at) + SELECT p.id, 'unassigned', 'auto_imported', NOW(), NOW() + FROM packages p + WHERE p.id = ANY($(packageIds)::bigint[]) + AND p.is_critical = true + ON CONFLICT (package_id) DO NOTHING + RETURNING 1 + ) + SELECT COUNT(*) AS count FROM ins + `, + { packageIds }, + ) + return parseInt(result.count, 10) +} From 1bcf9ad14ff6a1250cd5a7d26054e18fd401ac5d Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Wed, 10 Jun 2026 20:24:44 +0200 Subject: [PATCH 5/9] feat: add packages api Signed-off-by: Umberto Sgueglia --- .../config/custom-environment-variables.json | 7 + .../public/v1/packages/batchGetStewardship.ts | 36 +-- .../src/api/public/v1/packages/getPackage.ts | 80 +++++- .../public/v1/packages/getPackagesMetrics.ts | 10 +- .../api/public/v1/packages/listPackages.ts | 60 ++--- backend/src/conf/index.ts | 4 + backend/src/db/packagesDb.ts | 19 ++ services/libs/data-access-layer/src/index.ts | 1 + .../data-access-layer/src/osspckgs/api.ts | 227 ++++++++++++++++++ .../data-access-layer/src/osspckgs/index.ts | 1 + 10 files changed, 379 insertions(+), 66 deletions(-) create mode 100644 backend/src/db/packagesDb.ts create mode 100644 services/libs/data-access-layer/src/osspckgs/api.ts diff --git a/backend/config/custom-environment-variables.json b/backend/config/custom-environment-variables.json index 9aba188edf..4d1d01b6f5 100644 --- a/backend/config/custom-environment-variables.json +++ b/backend/config/custom-environment-variables.json @@ -49,6 +49,13 @@ "password": "PRODUCT_DB_PASSWORD", "database": "PRODUCT_DB_DATABASE" }, + "packagesDb": { + "host": "CROWD_PACKAGES_DB_WRITE_HOST", + "port": "CROWD_PACKAGES_DB_PORT", + "user": "CROWD_PACKAGES_DB_USERNAME", + "password": "CROWD_PACKAGES_DB_PASSWORD", + "database": "CROWD_PACKAGES_DB_DATABASE" + }, "segment": { "writeKey": "CROWD_SEGMENT_WRITE_KEY" }, diff --git a/backend/src/api/public/v1/packages/batchGetStewardship.ts b/backend/src/api/public/v1/packages/batchGetStewardship.ts index e888f1b5c8..ab02cf2160 100644 --- a/backend/src/api/public/v1/packages/batchGetStewardship.ts +++ b/backend/src/api/public/v1/packages/batchGetStewardship.ts @@ -1,11 +1,13 @@ import type { Request, Response } from 'express' import { z } from 'zod' +import { getPackagesByStewardshipPurls } from '@crowd/data-access-layer' + +import { getPackagesQx } from '@/db/packagesDb' import { ok } from '@/utils/api' import { validateOrThrow } from '@/utils/validation' -import { MOCK_DETAILS } from './mockData' -import type { OpenVulns, StewardshipSummary } from './types' +import type { StewardshipSummary } from './types' const MAX_PURLS = 100 @@ -16,29 +18,29 @@ const bodySchema = z.object({ .max(MAX_PURLS, `Maximum ${MAX_PURLS} purls per request`), }) -// TODO: replace with real DB queries once stewardship tables land export async function batchGetStewardship(req: Request, res: Response): Promise { const { purls } = validateOrThrow(bodySchema, req.body) + const qx = await getPackagesQx() + const rows = await getPackagesByStewardshipPurls(qx, purls) + + const byPurl = new Map(rows.map((r) => [r.purl, r])) + const packages: Record = {} for (const purl of purls) { - const detail = MOCK_DETAILS[purl] - if (!detail) { + const row = byPurl.get(purl) + if (!row) { packages[purl] = null } else { - const openVulns: OpenVulns = { low: 0, medium: 0, high: 0, critical: 0 } - for (const advisory of detail.security.advisories) { - openVulns[advisory.severity] += 1 - } packages[purl] = { - name: detail.name, - ecosystem: detail.ecosystem, - lifecycle: detail.general.riskSignals.lifecycle, - health: detail.general.healthScore.total, - impact: detail.general.impact.impactScore, - openVulns, - stewardship: detail.stewardship.status, - stewards: detail.stewardship.stewards, + name: row.name, + ecosystem: row.ecosystem, + lifecycle: null, + health: null, + impact: row.criticalityScore != null ? Math.round(Number(row.criticalityScore)) : null, + openVulns: null, + stewardship: (row.stewardshipStatus ?? 'unassigned') as StewardshipSummary['stewardship'], + stewards: null, lastActivityAt: null, lastActivityDescription: null, } diff --git a/backend/src/api/public/v1/packages/getPackage.ts b/backend/src/api/public/v1/packages/getPackage.ts index 31ababa00f..234cf97f38 100644 --- a/backend/src/api/public/v1/packages/getPackage.ts +++ b/backend/src/api/public/v1/packages/getPackage.ts @@ -1,29 +1,89 @@ import type { Request, Response } from 'express' import { z } from 'zod' -import { BadRequestError, NotFoundError } from '@crowd/common' +import { NotFoundError } from '@crowd/common' +import { getAdvisoriesByPackageId, getPackageDetailByPurl } from '@crowd/data-access-layer' +import { getPackagesQx } from '@/db/packagesDb' import { ok } from '@/utils/api' import { validateOrThrow } from '@/utils/validation' -import { MOCK_DETAILS } from './mockData' +import type { StewardshipStatus } from './types' const querySchema = z.object({ - purl: z.string().trim().min(1), + purl: z + .string() + .trim() + .min(1) + .refine((v) => v.startsWith('pkg:'), { message: 'purl must start with pkg:' }), }) -// TODO: replace with real DB queries once packages DB is wired into the backend export async function getPackage(req: Request, res: Response): Promise { const { purl } = validateOrThrow(querySchema, req.query) - if (!purl.startsWith('pkg:')) { - throw new BadRequestError('Invalid purl format: must start with pkg:') - } + const qx = await getPackagesQx() + const pkg = await getPackageDetailByPurl(qx, purl) - const detail = MOCK_DETAILS[purl] - if (!detail) { + if (!pkg) { throw new NotFoundError() } - ok(res, detail) + const advisories = await getAdvisoriesByPackageId(qx, pkg.id) + + ok(res, { + purl: pkg.purl, + name: pkg.name, + ecosystem: pkg.ecosystem, + general: { + healthScore: null, + impact: { + impactScore: pkg.criticalityScore != null ? Math.round(Number(pkg.criticalityScore)) : null, + downloadsLastMonth: + pkg.downloadsLast30d != null ? parseInt(pkg.downloadsLast30d, 10) : null, + dependentPackages: pkg.dependentPackagesCount ?? null, + dependentRepos: pkg.dependentReposCount ?? null, + transitiveReach: null, + }, + riskSignals: { + lifecycle: null, + maintainerBusFactor: null, + lastRelease: pkg.latestReleaseAt ? pkg.latestReleaseAt.toISOString() : null, + hasSecurityFile: pkg.hasSecurityFile ?? null, + openSSFScorecard: pkg.scorecardScore != null ? Number(pkg.scorecardScore) : null, + }, + }, + assessment: {}, + security: { + securityContacts: null, + advisories: advisories.map((a) => ({ + osvId: a.osvId, + severity: a.severity, + resolution: null, + })), + cvd: { + isPvrEnabled: null, + hasSecurityPolicyEnabled: pkg.hasSecurityPolicy ?? null, + tier0Steward: null, + criticalVulnerabilityFlag: pkg.hasCriticalVulnerability, + }, + }, + provenance: { + repositoryMapping: { + declaredRepo: pkg.repoUrl ?? pkg.repositoryUrl ?? pkg.declaredRepositoryUrl ?? null, + mappingConfidence: + pkg.repoMappingConfidence != null ? Number(pkg.repoMappingConfidence) : null, + lastCommitAt: pkg.repoLastCommitAt ? pkg.repoLastCommitAt.toISOString() : null, + }, + supplyChainIntegrity: { + buildProvenance: null, + signedReleases: null, + }, + }, + stewardship: { + status: (pkg.stewardshipStatus ?? 'unassigned') as StewardshipStatus, + stewards: null, + lastActivityAt: null, + }, + history: {}, + }) } diff --git a/backend/src/api/public/v1/packages/getPackagesMetrics.ts b/backend/src/api/public/v1/packages/getPackagesMetrics.ts index 89f12f2a46..df1b3e5595 100644 --- a/backend/src/api/public/v1/packages/getPackagesMetrics.ts +++ b/backend/src/api/public/v1/packages/getPackagesMetrics.ts @@ -1,10 +1,12 @@ import type { Request, Response } from 'express' -import { ok } from '@/utils/api' +import { getPackageMetrics } from '@crowd/data-access-layer' -import { MOCK_METRICS } from './mockData' +import { getPackagesQx } from '@/db/packagesDb' +import { ok } from '@/utils/api' -// TODO: replace with real DB queries once packages DB is wired into the backend export async function getPackagesMetrics(req: Request, res: Response): Promise { - ok(res, MOCK_METRICS) + const qx = await getPackagesQx() + const metrics = await getPackageMetrics(qx) + ok(res, metrics) } diff --git a/backend/src/api/public/v1/packages/listPackages.ts b/backend/src/api/public/v1/packages/listPackages.ts index 6e912c3581..05538753bd 100644 --- a/backend/src/api/public/v1/packages/listPackages.ts +++ b/backend/src/api/public/v1/packages/listPackages.ts @@ -1,14 +1,16 @@ import type { Request, Response } from 'express' import { z } from 'zod' +import { listPackagesForApi } from '@crowd/data-access-layer' + +import { getPackagesQx } from '@/db/packagesDb' import { ok } from '@/utils/api' import { validateOrThrow } from '@/utils/validation' -import { MOCK_DETAILS, MOCK_PACKAGES } from './mockData' +import type { StewardshipStatus } from './types' const DEFAULT_PAGE_SIZE = 20 const MAX_PAGE_SIZE = 100 -const STALE_MONTHS = 18 const booleanQueryParam = z.preprocess((v) => v === 'true', z.boolean()).default(false) @@ -26,7 +28,6 @@ const querySchema = z.object({ sortDir: z.enum(['asc', 'desc']).default('asc'), }) -// TODO: replace with real DB queries once packages DB is wired into the backend export async function listPackages(req: Request, res: Response): Promise { const { page, @@ -40,40 +41,29 @@ export async function listPackages(req: Request, res: Response): Promise { sortDir, } = validateOrThrow(querySchema, req.query) - const staleThreshold = new Date() - staleThreshold.setMonth(staleThreshold.getMonth() - STALE_MONTHS) - - let filtered = MOCK_PACKAGES.filter((p) => { - if (ecosystem && p.ecosystem !== ecosystem) return false - if (lifecycle && p.lifecycle !== lifecycle) return false - if (busFactor1Only && p.maintainerBusFactor !== 1) return false - if (unstewardedOnly && p.stewardship !== null && p.stewardship !== 'unassigned') return false - if (staleOnly) { - const lastRelease = MOCK_DETAILS[p.purl]?.general.riskSignals.lastRelease - if (!lastRelease || new Date(lastRelease) >= staleThreshold) return false - } - return true - }) - - filtered = filtered.sort((a, b) => { - let cmp = 0 - if (sortBy === 'name') { - cmp = a.name.localeCompare(b.name) - } else if (sortBy === 'health') { - cmp = (a.health ?? 0) - (b.health ?? 0) - } else if (sortBy === 'impact') { - cmp = (a.impact ?? 0) - (b.impact ?? 0) - } else if (sortBy === 'openVulns') { - const sumA = a.openVulns.low + a.openVulns.medium + a.openVulns.high + a.openVulns.critical - const sumB = b.openVulns.low + b.openVulns.medium + b.openVulns.high + b.openVulns.critical - cmp = sumA - sumB - } - return sortDir === 'desc' ? -cmp : cmp + const qx = await getPackagesQx() + const { rows, total } = await listPackagesForApi(qx, { + page, + pageSize, + ecosystem, + staleOnly, + unstewardedOnly, + sortBy, + sortDir, }) - const total = filtered.length - const start = (page - 1) * pageSize - const packages = filtered.slice(start, start + pageSize) + const packages = rows.map((r) => ({ + purl: r.purl, + name: r.name, + ecosystem: r.ecosystem, + health: null, + impact: r.criticalityScore != null ? Math.round(Number(r.criticalityScore)) : null, + lifecycle: null, + maintainerBusFactor: null, + openVulns: null, + stewardship: (r.stewardshipStatus ?? 'unassigned') as StewardshipStatus, + stewards: null, + })) ok(res, { page, diff --git a/backend/src/conf/index.ts b/backend/src/conf/index.ts index dfebb3c4ea..553167cde5 100644 --- a/backend/src/conf/index.ts +++ b/backend/src/conf/index.ts @@ -82,6 +82,10 @@ export const PRODUCT_DB_CONFIG: IDatabaseConfig = config.has('productDb') ? config.get('productDb') : undefined +export const PACKAGES_DB_CONFIG: IDatabaseConfig | undefined = config.has('packagesDb') + ? config.get('packagesDb') + : undefined + export const SEGMENT_CONFIG: SegmentConfiguration = config.get('segment') export const COMPREHEND_CONFIG: ComprehendConfiguration = diff --git a/backend/src/db/packagesDb.ts b/backend/src/db/packagesDb.ts new file mode 100644 index 0000000000..23c9aa09a2 --- /dev/null +++ b/backend/src/db/packagesDb.ts @@ -0,0 +1,19 @@ +import { getDbConnection } from '@crowd/data-access-layer/src/database' +import { QueryExecutor, pgpQx } from '@crowd/data-access-layer/src/queryExecutor' + +import { PACKAGES_DB_CONFIG } from '@/conf' + +let _qx: QueryExecutor | undefined + +export async function getPackagesQx(): Promise { + if (!_qx) { + if (!PACKAGES_DB_CONFIG) { + throw new Error( + 'Packages DB is not configured — set CROWD_PACKAGES_DB_* environment variables', + ) + } + const conn = await getDbConnection(PACKAGES_DB_CONFIG) + _qx = pgpQx(conn) + } + return _qx +} diff --git a/services/libs/data-access-layer/src/index.ts b/services/libs/data-access-layer/src/index.ts index 1070d84c1c..95b8cba162 100644 --- a/services/libs/data-access-layer/src/index.ts +++ b/services/libs/data-access-layer/src/index.ts @@ -22,3 +22,4 @@ export * from './osspckgs/packages' export * from './osspckgs/repos' export * from './osspckgs/stewardships' export * from './osspckgs/versions' +export * from './osspckgs/api' diff --git a/services/libs/data-access-layer/src/osspckgs/api.ts b/services/libs/data-access-layer/src/osspckgs/api.ts new file mode 100644 index 0000000000..1a5de505e1 --- /dev/null +++ b/services/libs/data-access-layer/src/osspckgs/api.ts @@ -0,0 +1,227 @@ +import { QueryExecutor } from '../queryExecutor' + +export interface PackageMetrics { + totalPackages: number + criticalPackages: number +} + +export async function getPackageMetrics(qx: QueryExecutor): Promise { + const row: { total: string; critical: string } = await qx.selectOne(` + SELECT + COUNT(*) AS total, + COUNT(*) FILTER (WHERE has_critical_vulnerability = true) AS critical + FROM packages + WHERE is_critical = true + `) + return { + totalPackages: parseInt(row.total, 10), + criticalPackages: parseInt(row.critical, 10), + } +} + +export interface PackageStewardshipRow { + purl: string + name: string + ecosystem: string + criticalityScore: number | null + stewardshipStatus: string | null +} + +export async function getPackagesByStewardshipPurls( + qx: QueryExecutor, + purls: string[], +): Promise { + if (purls.length === 0) return [] + return qx.select( + ` + SELECT + p.purl, + p.name, + p.ecosystem, + p.impact AS "criticalityScore", + s.status AS "stewardshipStatus" + FROM packages p + LEFT JOIN stewardships s ON s.package_id = p.id + WHERE p.purl = ANY($(purls)) + `, + { purls }, + ) +} + +export interface PackageListRow { + purl: string + name: string + ecosystem: string + criticalityScore: number | null + stewardshipStatus: string | null + latestReleaseAt: Date | null + total: string +} + +export interface ListPackagesOptions { + page: number + pageSize: number + ecosystem?: string + staleOnly: boolean + unstewardedOnly: boolean + sortBy: 'name' | 'health' | 'impact' | 'openVulns' + sortDir: 'asc' | 'desc' +} + +const STALE_MONTHS = 18 + +export async function listPackagesForApi( + qx: QueryExecutor, + opts: ListPackagesOptions, +): Promise<{ rows: PackageListRow[]; total: number }> { + const conditions: string[] = ['p.is_critical = true'] + const params: Record = {} + + if (opts.ecosystem) { + conditions.push('p.ecosystem = $(ecosystem)') + params.ecosystem = opts.ecosystem + } + + if (opts.staleOnly) { + conditions.push( + `(p.latest_release_at IS NULL OR p.latest_release_at < NOW() - INTERVAL '${STALE_MONTHS} months')`, + ) + } + + if (opts.unstewardedOnly) { + conditions.push(`(s.status = 'unassigned' OR s.id IS NULL)`) + } + + const where = `WHERE ${conditions.join(' AND ')}` + + // health, openVulns are v2 fields — fall back to name sort + const sortExpr = opts.sortBy === 'impact' ? 'p.impact' : 'LOWER(p.name)' + const sortDir = opts.sortDir === 'desc' ? 'DESC' : 'ASC' + + params.limit = opts.pageSize + params.offset = (opts.page - 1) * opts.pageSize + + const rows: PackageListRow[] = await qx.select( + ` + SELECT + p.purl, + p.name, + p.ecosystem, + p.impact AS "criticalityScore", + p.latest_release_at AS "latestReleaseAt", + s.status AS "stewardshipStatus", + COUNT(*) OVER() AS total + FROM packages p + LEFT JOIN stewardships s ON s.package_id = p.id + ${where} + ORDER BY ${sortExpr} ${sortDir} NULLS LAST + LIMIT $(limit) OFFSET $(offset) + `, + params, + ) + + const total = rows.length > 0 ? parseInt(rows[0].total, 10) : 0 + return { rows, total } +} + +export interface PackageDetailRow { + id: string + purl: string + name: string + ecosystem: string + criticalityScore: number | null + dependentPackagesCount: number | null + dependentReposCount: number | null + latestVersion: string | null + versionsCount: number | null + latestReleaseAt: Date | null + declaredRepositoryUrl: string | null + repositoryUrl: string | null + hasCriticalVulnerability: boolean + stewardshipStatus: string | null + stewardshipLastStatusAt: Date | null + // from package_repos + repos + repoUrl: string | null + repoMappingConfidence: number | null + repoLastCommitAt: Date | null + scorecardScore: number | null + hasSecurityFile: boolean | null + hasSecurityPolicy: boolean | null + // from downloads_last_30d + downloadsLast30d: string | null +} + +export interface AdvisoryRow { + osvId: string + severity: string +} + +export async function getPackageDetailByPurl( + qx: QueryExecutor, + purl: string, +): Promise { + return qx.selectOneOrNone( + ` + SELECT + p.id::text AS id, + p.purl, + p.name, + p.ecosystem, + p.impact AS "criticalityScore", + p.dependent_count AS "dependentPackagesCount", + p.dependent_repos_count AS "dependentReposCount", + p.latest_version AS "latestVersion", + p.versions_count AS "versionsCount", + p.latest_release_at AS "latestReleaseAt", + p.declared_repository_url AS "declaredRepositoryUrl", + p.repository_url AS "repositoryUrl", + p.has_critical_vulnerability AS "hasCriticalVulnerability", + s.status AS "stewardshipStatus", + s.last_status_at AS "stewardshipLastStatusAt", + -- best repo link (highest confidence, prefer declared) + r.url AS "repoUrl", + pr.confidence AS "repoMappingConfidence", + r.last_commit_at AS "repoLastCommitAt", + r.scorecard_score AS "scorecardScore", + r.security_file_enabled AS "hasSecurityFile", + r.security_policy_enabled AS "hasSecurityPolicy", + -- latest 30-day download count + ( + SELECT d.count::text + FROM downloads_last_30d d + WHERE d.purl = p.purl + ORDER BY d.end_date DESC + LIMIT 1 + ) AS "downloadsLast30d" + FROM packages p + LEFT JOIN stewardships s ON s.package_id = p.id + LEFT JOIN LATERAL ( + SELECT pr2.repo_id, pr2.confidence + FROM package_repos pr2 + WHERE pr2.package_id = p.id + ORDER BY pr2.confidence DESC, (pr2.source = 'declared') DESC + LIMIT 1 + ) pr ON true + LEFT JOIN repos r ON r.id = pr.repo_id + WHERE p.purl = $(purl) + `, + { purl }, + ) +} + +export async function getAdvisoriesByPackageId( + qx: QueryExecutor, + packageId: string, +): Promise { + return qx.select( + ` + SELECT + a.osv_id AS "osvId", + LOWER(a.severity) AS severity + FROM advisory_packages ap + JOIN advisories a ON a.id = ap.advisory_id + WHERE ap.package_id = $(packageId)::bigint + `, + { packageId }, + ) +} diff --git a/services/libs/data-access-layer/src/osspckgs/index.ts b/services/libs/data-access-layer/src/osspckgs/index.ts index 65228ac4d7..5b3e9b3641 100644 --- a/services/libs/data-access-layer/src/osspckgs/index.ts +++ b/services/libs/data-access-layer/src/osspckgs/index.ts @@ -4,3 +4,4 @@ export * from './maintainers' export * from './versions' export * from './repos' export * from './stewardships' +export * from './api' From f0cebf0079ac78d429e4bbad7a42d2849c624b9a Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Wed, 10 Jun 2026 20:34:01 +0200 Subject: [PATCH 6/9] fix: db visibility Signed-off-by: Umberto Sgueglia --- backend/src/db/packagesDb.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/backend/src/db/packagesDb.ts b/backend/src/db/packagesDb.ts index 23c9aa09a2..f6445d2951 100644 --- a/backend/src/db/packagesDb.ts +++ b/backend/src/db/packagesDb.ts @@ -3,17 +3,16 @@ import { QueryExecutor, pgpQx } from '@crowd/data-access-layer/src/queryExecutor import { PACKAGES_DB_CONFIG } from '@/conf' -let _qx: QueryExecutor | undefined +let _init: Promise | undefined -export async function getPackagesQx(): Promise { - if (!_qx) { +export function getPackagesQx(): Promise { + if (!_init) { if (!PACKAGES_DB_CONFIG) { throw new Error( 'Packages DB is not configured — set CROWD_PACKAGES_DB_* environment variables', ) } - const conn = await getDbConnection(PACKAGES_DB_CONFIG) - _qx = pgpQx(conn) + _init = getDbConnection(PACKAGES_DB_CONFIG).then(pgpQx) } - return _qx + return _init } From 30661400628ad94d5dd1310e0739691bb2d10aaf Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Thu, 11 Jun 2026 12:38:54 +0200 Subject: [PATCH 7/9] fix: add null info which can be already fetched Signed-off-by: Umberto Sgueglia --- .../public/v1/packages/batchGetStewardship.ts | 3 +- .../src/api/public/v1/packages/getPackage.ts | 13 ++--- .../api/public/v1/packages/listPackages.ts | 11 ++-- backend/src/db/packagesDb.ts | 7 ++- .../data-access-layer/src/osspckgs/api.ts | 51 ++++++++++++++++--- 5 files changed, 67 insertions(+), 18 deletions(-) diff --git a/backend/src/api/public/v1/packages/batchGetStewardship.ts b/backend/src/api/public/v1/packages/batchGetStewardship.ts index ab02cf2160..d5f7490255 100644 --- a/backend/src/api/public/v1/packages/batchGetStewardship.ts +++ b/backend/src/api/public/v1/packages/batchGetStewardship.ts @@ -37,7 +37,8 @@ export async function batchGetStewardship(req: Request, res: Response): Promise< ecosystem: row.ecosystem, lifecycle: null, health: null, - impact: row.criticalityScore != null ? Math.round(Number(row.criticalityScore)) : null, + impact: + row.criticalityScore != null ? Math.round(Number(row.criticalityScore) * 100) : null, openVulns: null, stewardship: (row.stewardshipStatus ?? 'unassigned') as StewardshipSummary['stewardship'], stewards: null, diff --git a/backend/src/api/public/v1/packages/getPackage.ts b/backend/src/api/public/v1/packages/getPackage.ts index 234cf97f38..c0c7ce5dc4 100644 --- a/backend/src/api/public/v1/packages/getPackage.ts +++ b/backend/src/api/public/v1/packages/getPackage.ts @@ -37,18 +37,19 @@ export async function getPackage(req: Request, res: Response): Promise { general: { healthScore: null, impact: { - impactScore: pkg.criticalityScore != null ? Math.round(Number(pkg.criticalityScore)) : null, + impactScore: + pkg.criticalityScore != null ? Math.round(Number(pkg.criticalityScore) * 100) : null, downloadsLastMonth: pkg.downloadsLast30d != null ? parseInt(pkg.downloadsLast30d, 10) : null, dependentPackages: pkg.dependentPackagesCount ?? null, dependentRepos: pkg.dependentReposCount ?? null, - transitiveReach: null, + transitiveReach: pkg.transitiveReach, }, riskSignals: { lifecycle: null, - maintainerBusFactor: null, + maintainerBusFactor: pkg.maintainerCount, lastRelease: pkg.latestReleaseAt ? pkg.latestReleaseAt.toISOString() : null, - hasSecurityFile: pkg.hasSecurityFile ?? null, + hasSecurityFile: pkg.hasSecurityFile, openSSFScorecard: pkg.scorecardScore != null ? Number(pkg.scorecardScore) : null, }, }, @@ -58,11 +59,11 @@ export async function getPackage(req: Request, res: Response): Promise { advisories: advisories.map((a) => ({ osvId: a.osvId, severity: a.severity, - resolution: null, + resolution: a.resolution, })), cvd: { isPvrEnabled: null, - hasSecurityPolicyEnabled: pkg.hasSecurityPolicy ?? null, + hasSecurityPolicyEnabled: pkg.hasSecurityPolicy, tier0Steward: null, criticalVulnerabilityFlag: pkg.hasCriticalVulnerability, }, diff --git a/backend/src/api/public/v1/packages/listPackages.ts b/backend/src/api/public/v1/packages/listPackages.ts index 05538753bd..fa9d731376 100644 --- a/backend/src/api/public/v1/packages/listPackages.ts +++ b/backend/src/api/public/v1/packages/listPackages.ts @@ -41,6 +41,9 @@ export async function listPackages(req: Request, res: Response): Promise { sortDir, } = validateOrThrow(querySchema, req.query) + // health is a v2 field with no backing column yet — fall back to name sort + const effectiveSortBy = sortBy === 'health' ? 'name' : sortBy + const qx = await getPackagesQx() const { rows, total } = await listPackagesForApi(qx, { page, @@ -48,7 +51,7 @@ export async function listPackages(req: Request, res: Response): Promise { ecosystem, staleOnly, unstewardedOnly, - sortBy, + sortBy: effectiveSortBy, sortDir, }) @@ -57,10 +60,10 @@ export async function listPackages(req: Request, res: Response): Promise { name: r.name, ecosystem: r.ecosystem, health: null, - impact: r.criticalityScore != null ? Math.round(Number(r.criticalityScore)) : null, + impact: r.criticalityScore != null ? Math.round(Number(r.criticalityScore) * 100) : null, lifecycle: null, maintainerBusFactor: null, - openVulns: null, + openVulns: r.openVulns, stewardship: (r.stewardshipStatus ?? 'unassigned') as StewardshipStatus, stewards: null, })) @@ -76,7 +79,7 @@ export async function listPackages(req: Request, res: Response): Promise { staleOnly, unstewardedOnly, }, - sort: { by: sortBy, dir: sortDir }, + sort: { by: effectiveSortBy, dir: sortDir }, packages, }) } diff --git a/backend/src/db/packagesDb.ts b/backend/src/db/packagesDb.ts index f6445d2951..48ad1d5821 100644 --- a/backend/src/db/packagesDb.ts +++ b/backend/src/db/packagesDb.ts @@ -12,7 +12,12 @@ export function getPackagesQx(): Promise { 'Packages DB is not configured — set CROWD_PACKAGES_DB_* environment variables', ) } - _init = getDbConnection(PACKAGES_DB_CONFIG).then(pgpQx) + _init = getDbConnection(PACKAGES_DB_CONFIG) + .then(pgpQx) + .catch((err) => { + _init = undefined + throw err + }) } return _init } diff --git a/services/libs/data-access-layer/src/osspckgs/api.ts b/services/libs/data-access-layer/src/osspckgs/api.ts index 1a5de505e1..b24cc7c7e6 100644 --- a/services/libs/data-access-layer/src/osspckgs/api.ts +++ b/services/libs/data-access-layer/src/osspckgs/api.ts @@ -9,6 +9,7 @@ export async function getPackageMetrics(qx: QueryExecutor): Promise= '1.10.0' is TRUE here. + -- Replace with a proper semver comparison function when one is available in the DB. + WHEN BOOL_AND( + CASE + WHEN ar.fixed_version IS NULL AND ar.last_affected IS NULL THEN FALSE + WHEN ar.fixed_version IS NOT NULL AND p.latest_version >= ar.fixed_version THEN TRUE + WHEN ar.fixed_version IS NOT NULL THEN FALSE + WHEN ar.last_affected IS NOT NULL AND p.latest_version > ar.last_affected THEN TRUE + ELSE FALSE + END + ) THEN 'patched' + ELSE 'open' + END AS resolution FROM advisory_packages ap JOIN advisories a ON a.id = ap.advisory_id + LEFT JOIN advisory_affected_ranges ar ON ar.advisory_package_id = ap.id + JOIN packages p ON p.id = ap.package_id WHERE ap.package_id = $(packageId)::bigint + GROUP BY a.osv_id, a.severity, p.latest_version `, { packageId }, ) From dc9725870dc3c8dcb3d8d3aa0c6d196cd34b7a5f Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Thu, 11 Jun 2026 16:49:06 +0200 Subject: [PATCH 8/9] fix: optimize perf Signed-off-by: Umberto Sgueglia --- .../api/public/v1/packages/listPackages.ts | 4 +-- .../data-access-layer/src/osspckgs/api.ts | 26 +++++++++++-------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/backend/src/api/public/v1/packages/listPackages.ts b/backend/src/api/public/v1/packages/listPackages.ts index fa9d731376..60dc7fdcb3 100644 --- a/backend/src/api/public/v1/packages/listPackages.ts +++ b/backend/src/api/public/v1/packages/listPackages.ts @@ -20,8 +20,8 @@ const querySchema = z.object({ page: z.coerce.number().int().min(1).default(1), pageSize: z.coerce.number().int().min(1).max(MAX_PAGE_SIZE).default(DEFAULT_PAGE_SIZE), ecosystem: z.string().trim().optional(), - lifecycle: z.enum(lifecycleValues).optional(), - busFactor1Only: booleanQueryParam, + lifecycle: z.enum(lifecycleValues).optional(), // TODO: filter not yet implemented in DAL + busFactor1Only: booleanQueryParam, // TODO: filter not yet implemented in DAL staleOnly: booleanQueryParam, unstewardedOnly: booleanQueryParam, sortBy: z.enum(['name', 'health', 'impact', 'openVulns']).default('name'), diff --git a/services/libs/data-access-layer/src/osspckgs/api.ts b/services/libs/data-access-layer/src/osspckgs/api.ts index b24cc7c7e6..0ac46a9c14 100644 --- a/services/libs/data-access-layer/src/osspckgs/api.ts +++ b/services/libs/data-access-layer/src/osspckgs/api.ts @@ -115,10 +115,13 @@ export async function listPackagesForApi( p.impact AS "criticalityScore", p.latest_release_at AS "latestReleaseAt", s.status AS "stewardshipStatus", - (SELECT COUNT(*)::int FROM advisory_packages ap WHERE ap.package_id = p.id) AS "openVulns", + COALESCE(ap_counts.cnt, 0) AS "openVulns", COUNT(*) OVER() AS total FROM packages p LEFT JOIN stewardships s ON s.package_id = p.id + LEFT JOIN LATERAL ( + SELECT COUNT(*)::int AS cnt FROM advisory_packages WHERE package_id = p.id + ) ap_counts ON true ${where} ORDER BY ${sortExpr} ${sortDir} NULLS LAST, p.purl ${sortDir} LIMIT $(limit) OFFSET $(offset) @@ -203,16 +206,17 @@ export async function getPackageDetailByPurl( LIMIT 1 ) AS "downloadsLast30d", (SELECT COUNT(*)::int FROM package_maintainers pm WHERE pm.package_id = p.id) AS "maintainerCount", - -- percentile rank within ecosystem (0=least, 1=most transitive reach) - ( - SELECT r.prank - FROM ( - SELECT purl, PERCENT_RANK() OVER (PARTITION BY ecosystem ORDER BY transitive_dependent_count ASC NULLS FIRST) AS prank - FROM packages - WHERE ecosystem = p.ecosystem - ) r - WHERE r.purl = p.purl - ) AS "transitiveReach" + -- TODO: precompute and store in packages.transitive_reach_prank; full window scan is too slow at npm scale (~24s for npm) + -- ( + -- SELECT r.prank + -- FROM ( + -- SELECT purl, PERCENT_RANK() OVER (PARTITION BY ecosystem ORDER BY transitive_dependent_count ASC NULLS FIRST) AS prank + -- FROM packages + -- WHERE ecosystem = p.ecosystem + -- ) r + -- WHERE r.purl = p.purl + -- ) AS "transitiveReach" + NULL::float AS "transitiveReach" FROM packages p LEFT JOIN stewardships s ON s.package_id = p.id LEFT JOIN LATERAL ( From d353e703fcee7fd4f8122345a0b09eab5b12452c Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Thu, 11 Jun 2026 17:15:50 +0200 Subject: [PATCH 9/9] fix: add branch protection enabled Signed-off-by: Umberto Sgueglia --- backend/src/api/public/v1/packages/getPackage.ts | 2 +- backend/src/api/public/v1/packages/listPackages.ts | 2 +- services/libs/data-access-layer/src/osspckgs/api.ts | 10 ++++++---- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/backend/src/api/public/v1/packages/getPackage.ts b/backend/src/api/public/v1/packages/getPackage.ts index c0c7ce5dc4..4fd851dc9a 100644 --- a/backend/src/api/public/v1/packages/getPackage.ts +++ b/backend/src/api/public/v1/packages/getPackage.ts @@ -63,7 +63,7 @@ export async function getPackage(req: Request, res: Response): Promise { })), cvd: { isPvrEnabled: null, - hasSecurityPolicyEnabled: pkg.hasSecurityPolicy, + hasSecurityPolicyEnabled: pkg.branchProtectionEnabled, tier0Steward: null, criticalVulnerabilityFlag: pkg.hasCriticalVulnerability, }, diff --git a/backend/src/api/public/v1/packages/listPackages.ts b/backend/src/api/public/v1/packages/listPackages.ts index 60dc7fdcb3..3e07ca3b40 100644 --- a/backend/src/api/public/v1/packages/listPackages.ts +++ b/backend/src/api/public/v1/packages/listPackages.ts @@ -62,7 +62,7 @@ export async function listPackages(req: Request, res: Response): Promise { health: null, impact: r.criticalityScore != null ? Math.round(Number(r.criticalityScore) * 100) : null, lifecycle: null, - maintainerBusFactor: null, + maintainerBusFactor: r.maintainerCount, openVulns: r.openVulns, stewardship: (r.stewardshipStatus ?? 'unassigned') as StewardshipStatus, stewards: null, diff --git a/services/libs/data-access-layer/src/osspckgs/api.ts b/services/libs/data-access-layer/src/osspckgs/api.ts index 0ac46a9c14..84903d8e25 100644 --- a/services/libs/data-access-layer/src/osspckgs/api.ts +++ b/services/libs/data-access-layer/src/osspckgs/api.ts @@ -55,8 +55,8 @@ export interface PackageListRow { ecosystem: string criticalityScore: number | null stewardshipStatus: string | null - latestReleaseAt: Date | null - openVulns: number | null + openVulns: number + maintainerCount: number total: string } @@ -113,9 +113,9 @@ export async function listPackagesForApi( p.name, p.ecosystem, p.impact AS "criticalityScore", - p.latest_release_at AS "latestReleaseAt", s.status AS "stewardshipStatus", COALESCE(ap_counts.cnt, 0) AS "openVulns", + (SELECT COUNT(*)::int FROM package_maintainers pm WHERE pm.package_id = p.id) AS "maintainerCount", COUNT(*) OVER() AS total FROM packages p LEFT JOIN stewardships s ON s.package_id = p.id @@ -156,9 +156,10 @@ export interface PackageDetailRow { scorecardScore: number | null hasSecurityFile: boolean | null hasSecurityPolicy: boolean | null + branchProtectionEnabled: boolean | null // from downloads_last_30d downloadsLast30d: string | null - maintainerCount: number | null + maintainerCount: number transitiveReach: number | null } @@ -197,6 +198,7 @@ export async function getPackageDetailByPurl( r.scorecard_score AS "scorecardScore", r.security_file_enabled AS "hasSecurityFile", r.security_policy_enabled AS "hasSecurityPolicy", + r.branch_protection_enabled AS "branchProtectionEnabled", -- latest 30-day download count ( SELECT d.count::text