fix: merge case-duplicate github repos and enforce unique lower(url) [CM-1344] (#4383)

Signed-off-by: Mouad BANI <mouad-mb@outlook.com>

Author: mbani01

backend/src/osspckgs/migrations/V1784718693__merge_case_duplicate_github_repos.sql

@@ -0,0 +1,116 @@
+-- GitHub paths are case-insensitive, but repos.url is UNIQUE case-sensitively.
+-- Writers that predate URL lowercasing (cargo initial sync on 2026-06-19, maven
+-- enrichment before CM-1305) inserted mixed-case variants of rows that already
+-- existed lowercase: ~40k duplicate groups, ~10k of them serving duplicate
+-- security contacts from both variants. The CM-1305 backfills re-pointed
+-- package links to the lowercase rows but left the stale variants behind.
+--
+-- Keeper per group: the all-lowercase row when present (links were already
+-- re-pointed to it), else the lowest id. Only package_repos is primary data
+-- and gets re-pointed (repo_docker comes along since it is a free UPDATE);
+-- contacts, snapshots, and scorecard rows on losers are derived and re-fill
+-- via the regular sweeps, so they are dropped with the loser rows.
+
+CREATE TEMP TABLE repo_merge_members AS
+SELECT id AS repo_id, keeper_id, id = keeper_id AS is_keeper
+FROM (
+    SELECT
+        id,
+        FIRST_VALUE(id) OVER (
+            PARTITION BY LOWER(url)
+            ORDER BY (url = LOWER(url)) DESC, id
+        ) AS keeper_id,
+        COUNT(*) OVER (PARTITION BY LOWER(url)) AS group_size
+    FROM repos
+    WHERE host = 'github'
+) grouped
+WHERE group_size > 1;
+
+CREATE INDEX ON repo_merge_members (repo_id);
+ANALYZE repo_merge_members;
+
+-- Keep one link per (package, group), ranked the way consumers pick links
+-- (sqlFragments bestRepoLink: confidence DESC, declared-on-ties) so the merge
+-- preserves the strongest provenance; keeper status only breaks full ties.
+-- ROW_NUMBER instead of an EXISTS check against the keeper because 3-variant
+-- groups exist: two losers linking the same package would collide on
+-- UNIQUE (package_id, repo_id) after the re-point below.
+DELETE FROM package_repos
+WHERE id IN (
+    SELECT id
+    FROM (
+        SELECT pr.id,
+               ROW_NUMBER() OVER (
+                   PARTITION BY m.keeper_id, pr.package_id
+                   ORDER BY pr.confidence DESC, (pr.source = 'declared') DESC,
+                            m.is_keeper DESC, pr.verified_at DESC, pr.id
+               ) AS rn
+        FROM package_repos pr
+        JOIN repo_merge_members m ON m.repo_id = pr.repo_id
+    ) ranked
+    WHERE rn > 1
+);
+
+-- Re-point surviving loser links via insert + delete, never an UPDATE:
+-- repo_id is part of the Tinybird packageRepos sorting key (CDC models key
+-- changes as delete + insert; a key-mutating UPDATE would leave both links
+-- live under FINAL with no way to reconcile). The fresh row streams cleanly
+-- with a new id and a NOW() version; the stale loser keys (these deletes
+-- included) are removed by the post-migration Tinybird datasource rebuild.
+INSERT INTO package_repos (package_id, repo_id, source, confidence, verified_at, created_at)
+SELECT pr.package_id, m.keeper_id, pr.source, pr.confidence, NOW(), NOW()
+FROM package_repos pr
+JOIN repo_merge_members m ON m.repo_id = pr.repo_id
+WHERE NOT m.is_keeper;
+
+DELETE FROM package_repos pr
+USING repo_merge_members m
+WHERE pr.repo_id = m.repo_id
+  AND NOT m.is_keeper;
+
+UPDATE repo_docker d
+SET repo_id = m.keeper_id
+FROM repo_merge_members m
+WHERE d.repo_id = m.repo_id
+  AND NOT m.is_keeper;
+
+DELETE FROM repo_scorecard_checks c
+USING repo_merge_members m
+WHERE c.repo_id = m.repo_id
+  AND NOT m.is_keeper;
+
+-- packages.repository_url is denormalized and must keep matching the
+-- canonical repos.url (the maven backfill updates the two atomically for the
+-- same reason). Joined on repos directly, not the merge map, so it also
+-- covers packages pointing at mixed-case singletons normalized below; runs
+-- while loser urls still exist to match against. last_synced_at is the
+-- packages watermark, bumping it ships the correction to Tinybird.
+UPDATE packages p
+SET repository_url = LOWER(p.repository_url), last_synced_at = NOW()
+FROM repos r
+WHERE r.host = 'github'
+  AND p.repository_url = r.url
+  AND p.repository_url <> LOWER(p.repository_url);
+
+-- Cascades security_contacts and repo_activity_snapshot on losers.
+DELETE FROM repos r
+USING repo_merge_members m
+WHERE r.id = m.repo_id
+  AND NOT m.is_keeper;
+
+-- Lowercase every surviving mixed-case github url: keepers whose group had
+-- no lowercase variant, plus mixed-case singletons that never had a
+-- duplicate. Singletons must be normalized before the guard index exists —
+-- writers upsert with ON CONFLICT (url), which does not arbitrate on the
+-- LOWER(url) index, so a later insert of the same repo's canonical lowercase
+-- url (all writers lowercase now) would raise unique_violation instead of
+-- upserting. Safe against UNIQUE (url): any two rows sharing LOWER(url) were
+-- a group above, and their losers are gone by now.
+UPDATE repos r
+SET url = LOWER(r.url), updated_at = NOW()
+WHERE r.host = 'github'
+  AND r.url <> LOWER(r.url);
+
+-- The recurrence-guard unique index lives in the next migration: it must be
+-- built CONCURRENTLY (repos takes continuous worker writes), which cannot run
+-- inside this transaction.

backend/src/osspckgs/migrations/V1784718694__repos_github_lower_url_guard.sql

@@ -0,0 +1,22 @@
+-- Recurrence guard for case-duplicate GitHub repos, split from the merge in
+-- V1784718693: CONCURRENTLY cannot run inside a transaction, and under
+-- flyway's mixed=true a shared file would demote the merge itself to
+-- autocommit, losing its atomicity. The concurrent build keeps repos writable
+-- for the workers during the scan, and doubles as verification of the merge —
+-- a surviving duplicate fails the build.
+--
+-- GitHub-only: the only host with both confirmed duplicates and guaranteed
+-- lowercase-on-write today (CASE_INSENSITIVE_HOSTS in canonicalizeRepoUrl
+-- also covers gitlab.com, whose merge is a follow-up). On other hosts case
+-- can be significant and writers do not normalize, so a wider index could
+-- reject legitimately distinct repos.
+--
+-- A failed concurrent build leaves an INVALID index behind; the DROP lets a
+-- re-run replace it, where IF NOT EXISTS would silently keep the broken one.
+-- Also CONCURRENTLY: a plain drop takes an ACCESS EXCLUSIVE lock on repos,
+-- stalling the workers on the retry path this exists for.
+DROP INDEX CONCURRENTLY IF EXISTS repos_github_lower_url_uq;
+
+CREATE UNIQUE INDEX CONCURRENTLY repos_github_lower_url_uq
+    ON repos (LOWER(url))
+    WHERE host = 'github';