Merge branch 'main' into fix/cm-1216-email-domain-affiliations

Author: skwowet

backend/.env.dist.local

@@ -185,9 +185,9 @@ ENRICHER_BATCH_SIZE=100
 ENRICHER_REPO_UPDATE_INTERVAL_HOURS=24
 ENRICHER_IDLE_SLEEP_SEC=60
 
-OSSPCKGS_GCP_PROJECT=
-OSSPCKGS_GCS_BUCKET=
-OSSPCKGS_GCP_CREDENTIALS_B64=
+OSSPCKGS_GCP_PROJECT=local-dev
+OSSPCKGS_GCS_BUCKET=local-dev
+OSSPCKGS_GCP_CREDENTIALS_B64=e30=
 
 # osv-sync (Temporal-scheduled; see services/apps/packages_worker/src/osv/schedule.ts)
 # OSV_ECOSYSTEMS uses OSV's canonical bucket case (npm lowercase, Maven titlecase) because
@@ -199,3 +199,14 @@ OSV_ECOSYSTEMS=npm,Maven
 OSV_TMP_DIR=/tmp/osv
 OSV_BATCH_SIZE=500
 OSV_DERIVE_BATCH_SIZE=1000
+
+# maven enricher
+
+MAVEN_FETCHER_BATCH_SIZE=2000
+MAVEN_FETCHER_CONCURRENCY=10
+MAVEN_FETCHER_NON_CRITICAL_BATCH_SIZE=500
+MAVEN_FETCHER_NON_CRITICAL_CONCURRENCY=20
+MAVEN_FETCHER_REFRESH_DAYS=1
+MAVEN_FETCHER_GROUP_DELAY_MS=100
+MAVEN_FETCHER_BASE_URL_BACKFILL=https://maven-central.storage-download.googleapis.com/maven2
+MAVEN_FETCHER_BASE_URL_INCREMENTAL=https://repo1.maven.org/maven2

backend/src/osspckgs/migrations/V1780600000__add_created_updated_at.sql

@@ -0,0 +1,79 @@
+-- Add created_at / updated_at to all packages-db tables for Tinybird sync watermarking.
+--
+-- Excluded tables and why:
+--   package_name_history          — event log; has changed_at
+--   audit_field_changes           — audit log; has logged_at
+--   osspckgs_ingest_jobs          — job tracking; has started_at / finished_at
+--   npm_worker_state              — state machine; has updated_at
+--   npm_package_state             — watermark columns serve the same purpose
+--   npm_package_universe_state    — watermark columns serve the same purpose
+--   package_criticality_spotlight — override config; has added_at
+--
+-- repos: created_at already exists (stores the GitHub repository creation date).
+-- last_synced_at is nullable (not set until the GitHub enricher runs), so updated_at
+-- is added as a non-nullable watermark for Tinybird sync. last_synced_at is kept as
+-- a separate enrichment-freshness signal.
+--
+-- packages, versions, repo_docker: last_synced_at already serves as updated_at.
+-- Only created_at is added to avoid duplicate semantics.
+--
+-- package_repos: verified_at already serves as updated_at. Only created_at is added.
+--
+-- Partitioned tables (versions, package_dependencies, downloads_daily, downloads_last_30d):
+-- PostgreSQL 12+ propagates new columns to all child partitions automatically —
+-- no per-partition ALTER needed.
+
+ALTER TABLE repos
+  ADD COLUMN IF NOT EXISTS updated_at timestamptz NOT NULL DEFAULT NOW();
+
+ALTER TABLE packages
+  ADD COLUMN IF NOT EXISTS created_at timestamptz NOT NULL DEFAULT NOW();
+
+ALTER TABLE package_funding_links
+  ADD COLUMN IF NOT EXISTS created_at timestamptz NOT NULL DEFAULT NOW(),
+  ADD COLUMN IF NOT EXISTS updated_at timestamptz NOT NULL DEFAULT NOW();
+
+ALTER TABLE versions
+  ADD COLUMN IF NOT EXISTS created_at timestamptz NOT NULL DEFAULT NOW();
+
+ALTER TABLE package_dependencies
+  ADD COLUMN IF NOT EXISTS created_at timestamptz NOT NULL DEFAULT NOW(),
+  ADD COLUMN IF NOT EXISTS updated_at timestamptz NOT NULL DEFAULT NOW();
+
+ALTER TABLE repo_scorecard_checks
+  ADD COLUMN IF NOT EXISTS created_at timestamptz NOT NULL DEFAULT NOW(),
+  ADD COLUMN IF NOT EXISTS updated_at timestamptz NOT NULL DEFAULT NOW();
+
+ALTER TABLE repo_docker
+  ADD COLUMN IF NOT EXISTS created_at timestamptz NOT NULL DEFAULT NOW();
+
+ALTER TABLE package_repos
+  ADD COLUMN IF NOT EXISTS created_at timestamptz NOT NULL DEFAULT NOW();
+
+ALTER TABLE advisories
+  ADD COLUMN IF NOT EXISTS created_at timestamptz NOT NULL DEFAULT NOW(),
+  ADD COLUMN IF NOT EXISTS updated_at timestamptz NOT NULL DEFAULT NOW();
+
+ALTER TABLE advisory_packages
+  ADD COLUMN IF NOT EXISTS created_at timestamptz NOT NULL DEFAULT NOW(),
+  ADD COLUMN IF NOT EXISTS updated_at timestamptz NOT NULL DEFAULT NOW();
+
+ALTER TABLE advisory_affected_ranges
+  ADD COLUMN IF NOT EXISTS created_at timestamptz NOT NULL DEFAULT NOW(),
+  ADD COLUMN IF NOT EXISTS updated_at timestamptz NOT NULL DEFAULT NOW();
+
+ALTER TABLE maintainers
+  ADD COLUMN IF NOT EXISTS created_at timestamptz NOT NULL DEFAULT NOW(),
+  ADD COLUMN IF NOT EXISTS updated_at timestamptz NOT NULL DEFAULT NOW();
+
+ALTER TABLE package_maintainers
+  ADD COLUMN IF NOT EXISTS created_at timestamptz NOT NULL DEFAULT NOW(),
+  ADD COLUMN IF NOT EXISTS updated_at timestamptz NOT NULL DEFAULT NOW();
+
+ALTER TABLE downloads_daily
+  ADD COLUMN IF NOT EXISTS created_at timestamptz NOT NULL DEFAULT NOW(),
+  ADD COLUMN IF NOT EXISTS updated_at timestamptz NOT NULL DEFAULT NOW();
+
+ALTER TABLE downloads_last_30d
+  ADD COLUMN IF NOT EXISTS created_at timestamptz NOT NULL DEFAULT NOW(),
+  ADD COLUMN IF NOT EXISTS updated_at timestamptz NOT NULL DEFAULT NOW();

backend/src/osspckgs/migrations/V1780926783__DropPackagesUniverse.sql

@@ -0,0 +1,170 @@
+-- Retire packages_universe (Tier 3 workspace).  All signals are migrated onto
+-- packages, rank_packages_universe() is replaced by rank_packages() which operates on packages directly,
+-- and the table is dropped.
+--
+-- Columns migrated from packages_universe → packages:
+--   downloads_last_30d   bigint         (npm 30-day window cache for ranking)
+--   centrality_score     numeric(10,8)  (PageRank, stored for future formula use)
+--   rank_in_ecosystem    int            (computed by rank_packages)
+--
+-- rank_packages_universe() → rank_packages() changes:
+--   - Operates on packages directly; no more TRUNCATE/INSERT workspace.
+--   - Scope limited to ecosystems present in critical_top_n_by_ecosystem JSONB
+--     (dynamic — add an ecosystem to the JSONB to include it in ranking).
+--   - "Propagate to packages" step removed (packages IS the target now).
+--   - Return column propagated_rows removed (no propagation step).
+
+-- ── 1. Add missing columns to packages ────────────────────────────────────────
+
+ALTER TABLE packages
+  ADD COLUMN IF NOT EXISTS downloads_last_30d  bigint,
+  ADD COLUMN IF NOT EXISTS centrality_score    numeric(10, 8),
+  ADD COLUMN IF NOT EXISTS rank_in_ecosystem   int;
+
+-- ── 2. Back-fill from packages_universe ───────────────────────────────────────
+
+UPDATE packages p
+   SET downloads_last_30d = pu.downloads_last_30d,
+       centrality_score   = pu.centrality_score,
+       rank_in_ecosystem  = pu.rank_in_ecosystem
+  FROM packages_universe pu
+ WHERE p.purl = pu.purl
+   AND (
+         p.downloads_last_30d IS DISTINCT FROM pu.downloads_last_30d
+      OR p.centrality_score   IS DISTINCT FROM pu.centrality_score
+      OR p.rank_in_ecosystem  IS DISTINCT FROM pu.rank_in_ecosystem
+   );
+
+-- ── 3. Replace rank_packages_universe() ───────────────────────────────────────
+-- Two overloads exist in the schema:
+--   V1779710880 created (numeric, numeric, numeric, numeric, jsonb) — 5 params
+--   V1780589607 CREATE OR REPLACE'd with (numeric, numeric, numeric, jsonb) — different
+--     signature, so it added a second overload rather than replacing the first.
+-- Both must be dropped; only one new 4-param version is created.
+
+DROP FUNCTION IF EXISTS rank_packages_universe(numeric, numeric, numeric, numeric, jsonb);
+DROP FUNCTION IF EXISTS rank_packages_universe(numeric, numeric, numeric, jsonb);
+
+-- Usage:
+--   -- with defaults (weights 0.25/0.25/0.50, built-in top-N budget)
+--   SELECT * FROM rank_packages();
+--
+--   -- with custom weights and/or a different top-N budget
+--   SELECT * FROM rank_packages(
+--     0.20, 0.30, 0.50,
+--     '{"npm": 400000, "maven": 200000, "cargo": 75000}'::jsonb
+--   );
+
+-- rank_packages() — score, rank, and flag packages in one pass.
+--
+-- Formula:
+--   impact = w_downloads  * pct_rank( LOG(1 + downloads_last_30d)              ) within ecosystem
+--           + w_dep_pkgs  * pct_rank( LOG(1 + dependent_count)            ) within ecosystem
+--           + w_transitive * pct_rank( LOG(1 + transitive_dependent_count) ) within ecosystem
+--
+-- Steps:
+--   1. Score    — compute impact via weighted PERCENT_RANK() (scoped to JSONB ecosystems)
+--   2. Rank     — ROW_NUMBER() per ecosystem, flag top-N as is_critical (scoped to JSONB ecosystems)
+--   2.5 Spotlight — force is_critical = TRUE for rows in package_criticality_spotlight
+--   3. Stamp    — unconditionally set last_rank_pass_at on all scored rows (schema contract)
+--
+-- All weights and the top-N budget are call-time parameters.
+-- ROW_NUMBER() (not RANK()) keeps each ecosystem's critical set exactly at top-N.
+-- Only ecosystems present as keys in critical_top_n_by_ecosystem are scored/ranked;
+-- packages from other ecosystems are not touched.
+
+CREATE OR REPLACE FUNCTION rank_packages(
+    weight_downloads            numeric DEFAULT 0.25,
+    weight_dependent_packages   numeric DEFAULT 0.25,
+    weight_transitive           numeric DEFAULT 0.50,
+    critical_top_n_by_ecosystem jsonb   DEFAULT '{"npm":400000,"go":100000,"maven":200000,"pypi":100000,"nuget":50000,"cargo":75000}'::jsonb
+)
+RETURNS TABLE(scored_rows int, ranked_rows int)
+LANGUAGE plpgsql AS $$
+DECLARE
+    n_scored int;
+    n_ranked int;
+BEGIN
+    -- ── Step 1: score ──────────────────────────────────────────────────────────
+    WITH percentile_scores AS (
+        SELECT
+            id,
+            (
+              weight_downloads * PERCENT_RANK() OVER (
+                  PARTITION BY ecosystem ORDER BY LOG(1 + COALESCE(downloads_last_30d, 0)))
+
+            + weight_dependent_packages * PERCENT_RANK() OVER (
+                  PARTITION BY ecosystem ORDER BY LOG(1 + COALESCE(dependent_count, 0)))
+
+            + weight_transitive * PERCENT_RANK() OVER (
+                  PARTITION BY ecosystem ORDER BY LOG(1 + COALESCE(transitive_dependent_count, 0)))
+            )::numeric(10, 4) AS new_impact
+        FROM packages
+        WHERE ecosystem IN (SELECT jsonb_object_keys(critical_top_n_by_ecosystem))
+    )
+    UPDATE packages p
+       SET impact = ps.new_impact
+      FROM percentile_scores ps
+     WHERE p.id = ps.id
+       AND p.impact IS DISTINCT FROM ps.new_impact;
+
+    GET DIAGNOSTICS n_scored = ROW_COUNT;
+
+    -- ── Step 2: rank + flag ────────────────────────────────────────────────────
+    WITH ranked AS (
+        SELECT
+            id, ecosystem,
+            ROW_NUMBER() OVER (
+                PARTITION BY ecosystem
+                ORDER BY impact DESC NULLS LAST, id
+            ) AS r
+        FROM packages
+        WHERE purl IS NOT NULL
+          AND ecosystem IN (SELECT jsonb_object_keys(critical_top_n_by_ecosystem))
+    ),
+    flagged AS (
+        SELECT
+            id, r,
+            COALESCE(
+                r <= (critical_top_n_by_ecosystem ->> ecosystem)::int,
+                FALSE
+            ) AS new_is_critical
+        FROM ranked
+    )
+    UPDATE packages p
+       SET rank_in_ecosystem = f.r,
+           is_critical       = f.new_is_critical
+      FROM flagged f
+     WHERE p.id = f.id
+       AND (
+             p.rank_in_ecosystem IS DISTINCT FROM f.r
+          OR p.is_critical       IS DISTINCT FROM f.new_is_critical
+       );
+
+    GET DIAGNOSTICS n_ranked = ROW_COUNT;
+
+    -- ── Step 2.5: spotlight overrides ─────────────────────────────────────────
+    UPDATE packages p
+       SET is_critical = TRUE
+      FROM package_criticality_spotlight s
+     WHERE p.ecosystem                    = s.ecosystem
+       AND (p.namespace IS NOT DISTINCT FROM s.namespace)
+       AND p.name                         = s.name
+       AND p.is_critical                  = FALSE;
+
+    -- ── Step 3: stamp last_rank_pass_at unconditionally ───────────────────────
+    -- Schema contract: must be updated on every pass (not only when scores change)
+    -- so stale-detection queries (last_rank_pass_at < NOW() - INTERVAL '8 days') work.
+    UPDATE packages
+       SET last_rank_pass_at = NOW()
+     WHERE ecosystem IN (SELECT jsonb_object_keys(critical_top_n_by_ecosystem));
+
+    RETURN QUERY SELECT n_scored, n_ranked;
+END;
+$$;
+
+-- ── 4. Drop packages_universe ─────────────────────────────────────────────────
+-- No FK constraints reference this table (npm_package_universe_state and
+-- downloads_last_30d use purl text, not FK).
+
+DROP TABLE packages_universe;

backend/src/osspckgs/migrations/V1780996561__repo_activity_snapshot.sql

@@ -0,0 +1,30 @@
+ALTER TABLE repos ADD COLUMN IF NOT EXISTS security_policy_enabled boolean;
+ALTER TABLE repos ADD COLUMN IF NOT EXISTS security_file_enabled boolean;
+ALTER TABLE repos ADD COLUMN IF NOT EXISTS snapshot_at timestamptz;
+
+CREATE TABLE IF NOT EXISTS repo_activity_snapshot (
+  repo_id                                   bigint PRIMARY KEY REFERENCES repos(id) ON DELETE CASCADE,
+  snapshot_at                               timestamptz NOT NULL,
+  window_months                             int NOT NULL DEFAULT 12,
+  -- commit activity
+  commits_last_12m                          int,
+  commits_last_6m                           int,
+  commits_prior_6m                          int,
+  -- PR health
+  prs_opened_last_12m                       int,
+  prs_merged_last_12m                       int,
+  prs_closed_unmerged_12m                   int,
+  pr_median_time_to_merge_hours             int,
+  pr_median_time_to_first_response_hours    int,
+  -- issue health
+  issues_opened_last_12m                    int,
+  issues_closed_last_12m                    int,
+  issues_opened_last_6m                     int,
+  issues_opened_prior_6m                    int,
+  issues_open_now                           int,
+  issue_median_time_to_close_hours          int,
+  issue_median_time_to_first_response_hours int
+);
+
+CREATE INDEX IF NOT EXISTS repo_activity_snapshot_snapshot_at_idx
+  ON repo_activity_snapshot (snapshot_at);

backend/src/osspckgs/migrations/V1781020800__merge_30d_into_npm_package_state.sql

@@ -0,0 +1,25 @@
+-- ── 1. Add the 30d watermark columns to npm_package_state ──────────────────────
+ALTER TABLE npm_package_state
+  ADD COLUMN IF NOT EXISTS downloads_30d_last_run_at           timestamptz,  -- breadth watermark: latest 30d window refreshed
+  ADD COLUMN IF NOT EXISTS downloads_30d_history_backfilled_at timestamptz,  -- depth watermark: NULL until full older history filled
+  ADD COLUMN IF NOT EXISTS downloads_30d_run_result            jsonb;        -- { status, httpStatus?, errorKind?, message? }
+
+-- Recreate the two indexes the old table had — both due-selection queries
+-- filter/order on these columns.
+CREATE INDEX IF NOT EXISTS npm_package_state_downloads_30d_last_run_at_idx
+  ON npm_package_state (downloads_30d_last_run_at);
+CREATE INDEX IF NOT EXISTS npm_package_state_downloads_30d_history_backfilled_at_idx
+  ON npm_package_state (downloads_30d_history_backfilled_at);
+
+-- ── 2. Migrate existing rows ───────────────────────────────────────────────────
+INSERT INTO npm_package_state
+  (purl, downloads_30d_last_run_at, downloads_30d_history_backfilled_at, downloads_30d_run_result)
+SELECT purl, downloads_30d_last_run_at, downloads_30d_history_backfilled_at, downloads_30d_run_result
+  FROM npm_package_universe_state
+ON CONFLICT (purl) DO UPDATE SET
+  downloads_30d_last_run_at           = EXCLUDED.downloads_30d_last_run_at,
+  downloads_30d_history_backfilled_at = EXCLUDED.downloads_30d_history_backfilled_at,
+  downloads_30d_run_result            = EXCLUDED.downloads_30d_run_result;
+
+-- ── 3. Drop the retired table ──────────────────────────────────────────────────
+DROP TABLE npm_package_universe_state;