From c918213f5779ee17115b63e2fe0a80ca7be20cd0 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Wed, 22 Jul 2026 12:40:36 +0100 Subject: [PATCH 1/4] fix: merge case-duplicate github repos and enforce lowercase url uniqueness Signed-off-by: Mouad BANI --- ...693__merge_case_duplicate_github_repos.sql | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 backend/src/osspckgs/migrations/V1784718693__merge_case_duplicate_github_repos.sql diff --git a/backend/src/osspckgs/migrations/V1784718693__merge_case_duplicate_github_repos.sql b/backend/src/osspckgs/migrations/V1784718693__merge_case_duplicate_github_repos.sql new file mode 100644 index 0000000000..897b5f4d10 --- /dev/null +++ b/backend/src/osspckgs/migrations/V1784718693__merge_case_duplicate_github_repos.sql @@ -0,0 +1,92 @@ +-- GitHub paths are case-insensitive, but repos.url is UNIQUE case-sensitively. +-- Writers that predate URL lowercasing (cargo initial sync on 2026-06-19, maven +-- enrichment before CM-1305) inserted mixed-case variants of rows that already +-- existed lowercase: ~40k duplicate groups, ~10k of them serving duplicate +-- security contacts from both variants. The CM-1305 backfills re-pointed +-- package links to the lowercase rows but left the stale variants behind. +-- +-- Keeper per group: the all-lowercase row when present (links were already +-- re-pointed to it), else the lowest id. Only package_repos is primary data +-- and gets re-pointed (repo_docker comes along since it is a free UPDATE); +-- contacts, snapshots, and scorecard rows on losers are derived and re-fill +-- via the regular sweeps, so they are dropped with the loser rows. + +CREATE TEMP TABLE repo_merge_members AS +SELECT id AS repo_id, keeper_id, id = keeper_id AS is_keeper +FROM ( + SELECT + id, + FIRST_VALUE(id) OVER ( + PARTITION BY LOWER(url) + ORDER BY (url = LOWER(url)) DESC, id + ) AS keeper_id, + COUNT(*) OVER (PARTITION BY LOWER(url)) AS group_size + FROM repos + WHERE host = 'github' +) grouped +WHERE group_size > 1; + +CREATE INDEX ON repo_merge_members (repo_id); +ANALYZE repo_merge_members; + +-- Keep one link per (package, group): the keeper's if it exists, else the most +-- recently verified. ROW_NUMBER instead of an EXISTS check against the keeper +-- because 3-variant groups exist: two losers linking the same package would +-- collide on UNIQUE (package_id, repo_id) after the re-point below. +DELETE FROM package_repos +WHERE id IN ( + SELECT id + FROM ( + SELECT pr.id, + ROW_NUMBER() OVER ( + PARTITION BY m.keeper_id, pr.package_id + ORDER BY m.is_keeper DESC, pr.verified_at DESC, pr.id + ) AS rn + FROM package_repos pr + JOIN repo_merge_members m ON m.repo_id = pr.repo_id + ) ranked + WHERE rn > 1 +); + +UPDATE package_repos pr +SET repo_id = m.keeper_id +FROM repo_merge_members m +WHERE pr.repo_id = m.repo_id + AND NOT m.is_keeper; + +UPDATE repo_docker d +SET repo_id = m.keeper_id +FROM repo_merge_members m +WHERE d.repo_id = m.repo_id + AND NOT m.is_keeper; + +DELETE FROM repo_scorecard_checks c +USING repo_merge_members m +WHERE c.repo_id = m.repo_id + AND NOT m.is_keeper; + +-- Cascades security_contacts and repo_activity_snapshot on losers. +DELETE FROM repos r +USING repo_merge_members m +WHERE r.id = m.repo_id + AND NOT m.is_keeper; + +-- Groups that had no lowercase variant: lowercase the surviving row. Safe +-- against UNIQUE (url) — every row sharing LOWER(url) was in the same group, +-- and its losers are gone by now. +UPDATE repos r +SET url = LOWER(r.url), updated_at = NOW() +FROM repo_merge_members m +WHERE r.id = m.repo_id + AND m.is_keeper + AND r.url <> LOWER(r.url); + +-- Recurrence guard; also fails the migration if any duplicate survived the +-- merge. GitHub-only: it is the only host with both confirmed duplicates and +-- guaranteed lowercase-on-write today (CASE_INSENSITIVE_HOSTS in +-- canonicalizeRepoUrl also covers gitlab.com, whose merge is a follow-up). +-- On other hosts case can be significant and writers do not normalize, so a +-- wider index could reject legitimately distinct repos. +CREATE UNIQUE INDEX IF NOT EXISTS repos_github_lower_url_uq + ON repos (LOWER(url)) + WHERE host = 'github'; From 8184f3c9c36365b745fe1af34c9ed62b17d0314c Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Wed, 22 Jul 2026 12:57:39 +0100 Subject: [PATCH 2/4] fix: code review Signed-off-by: Mouad BANI --- ...693__merge_case_duplicate_github_repos.sql | 37 ++++++++++++------- ...84718694__repos_github_lower_url_guard.sql | 20 ++++++++++ 2 files changed, 43 insertions(+), 14 deletions(-) create mode 100644 backend/src/osspckgs/migrations/V1784718694__repos_github_lower_url_guard.sql diff --git a/backend/src/osspckgs/migrations/V1784718693__merge_case_duplicate_github_repos.sql b/backend/src/osspckgs/migrations/V1784718693__merge_case_duplicate_github_repos.sql index 897b5f4d10..3091bfff3f 100644 --- a/backend/src/osspckgs/migrations/V1784718693__merge_case_duplicate_github_repos.sql +++ b/backend/src/osspckgs/migrations/V1784718693__merge_case_duplicate_github_repos.sql @@ -29,10 +29,12 @@ WHERE group_size > 1; CREATE INDEX ON repo_merge_members (repo_id); ANALYZE repo_merge_members; --- Keep one link per (package, group): the keeper's if it exists, else the most --- recently verified. ROW_NUMBER instead of an EXISTS check against the keeper --- because 3-variant groups exist: two losers linking the same package would --- collide on UNIQUE (package_id, repo_id) after the re-point below. +-- Keep one link per (package, group), ranked the way consumers pick links +-- (sqlFragments bestRepoLink: confidence DESC, declared-on-ties) so the merge +-- preserves the strongest provenance; keeper status only breaks full ties. +-- ROW_NUMBER instead of an EXISTS check against the keeper because 3-variant +-- groups exist: two losers linking the same package would collide on +-- UNIQUE (package_id, repo_id) after the re-point below. DELETE FROM package_repos WHERE id IN ( SELECT id @@ -40,7 +42,8 @@ WHERE id IN ( SELECT pr.id, ROW_NUMBER() OVER ( PARTITION BY m.keeper_id, pr.package_id - ORDER BY m.is_keeper DESC, pr.verified_at DESC, pr.id + ORDER BY pr.confidence DESC, (pr.source = 'declared') DESC, + m.is_keeper DESC, pr.verified_at DESC, pr.id ) AS rn FROM package_repos pr JOIN repo_merge_members m ON m.repo_id = pr.repo_id @@ -65,6 +68,18 @@ USING repo_merge_members m WHERE c.repo_id = m.repo_id AND NOT m.is_keeper; +-- packages.repository_url is denormalized and must keep matching the +-- canonical repos.url (the maven backfill updates the two atomically for the +-- same reason). Match against member urls while the loser rows still exist; +-- last_synced_at is the packages watermark, bumping it ships the correction +-- to Tinybird. +UPDATE packages p +SET repository_url = LOWER(p.repository_url), last_synced_at = NOW() +FROM repo_merge_members m +JOIN repos r ON r.id = m.repo_id +WHERE p.repository_url = r.url + AND p.repository_url <> LOWER(p.repository_url); + -- Cascades security_contacts and repo_activity_snapshot on losers. DELETE FROM repos r USING repo_merge_members m @@ -81,12 +96,6 @@ WHERE r.id = m.repo_id AND m.is_keeper AND r.url <> LOWER(r.url); --- Recurrence guard; also fails the migration if any duplicate survived the --- merge. GitHub-only: it is the only host with both confirmed duplicates and --- guaranteed lowercase-on-write today (CASE_INSENSITIVE_HOSTS in --- canonicalizeRepoUrl also covers gitlab.com, whose merge is a follow-up). --- On other hosts case can be significant and writers do not normalize, so a --- wider index could reject legitimately distinct repos. -CREATE UNIQUE INDEX IF NOT EXISTS repos_github_lower_url_uq - ON repos (LOWER(url)) - WHERE host = 'github'; +-- The recurrence-guard unique index lives in the next migration: it must be +-- built CONCURRENTLY (repos takes continuous worker writes), which cannot run +-- inside this transaction. diff --git a/backend/src/osspckgs/migrations/V1784718694__repos_github_lower_url_guard.sql b/backend/src/osspckgs/migrations/V1784718694__repos_github_lower_url_guard.sql new file mode 100644 index 0000000000..b99f542321 --- /dev/null +++ b/backend/src/osspckgs/migrations/V1784718694__repos_github_lower_url_guard.sql @@ -0,0 +1,20 @@ +-- Recurrence guard for case-duplicate GitHub repos, split from the merge in +-- V1784718693: CONCURRENTLY cannot run inside a transaction, and under +-- flyway's mixed=true a shared file would demote the merge itself to +-- autocommit, losing its atomicity. The concurrent build keeps repos writable +-- for the workers during the scan, and doubles as verification of the merge — +-- a surviving duplicate fails the build. +-- +-- GitHub-only: the only host with both confirmed duplicates and guaranteed +-- lowercase-on-write today (CASE_INSENSITIVE_HOSTS in canonicalizeRepoUrl +-- also covers gitlab.com, whose merge is a follow-up). On other hosts case +-- can be significant and writers do not normalize, so a wider index could +-- reject legitimately distinct repos. +-- +-- A failed concurrent build leaves an INVALID index behind; the DROP lets a +-- re-run replace it, where IF NOT EXISTS would silently keep the broken one. +DROP INDEX IF EXISTS repos_github_lower_url_uq; + +CREATE UNIQUE INDEX CONCURRENTLY repos_github_lower_url_uq + ON repos (LOWER(url)) + WHERE host = 'github'; From e3ab9725ea87d9773e94ec174b8a2f91ea2d33f9 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Wed, 22 Jul 2026 13:10:04 +0100 Subject: [PATCH 3/4] fix: code review Signed-off-by: Mouad BANI --- ...693__merge_case_duplicate_github_repos.sql | 28 +++++++++++-------- ...84718694__repos_github_lower_url_guard.sql | 4 ++- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/backend/src/osspckgs/migrations/V1784718693__merge_case_duplicate_github_repos.sql b/backend/src/osspckgs/migrations/V1784718693__merge_case_duplicate_github_repos.sql index 3091bfff3f..6cab95471c 100644 --- a/backend/src/osspckgs/migrations/V1784718693__merge_case_duplicate_github_repos.sql +++ b/backend/src/osspckgs/migrations/V1784718693__merge_case_duplicate_github_repos.sql @@ -70,14 +70,15 @@ WHERE c.repo_id = m.repo_id -- packages.repository_url is denormalized and must keep matching the -- canonical repos.url (the maven backfill updates the two atomically for the --- same reason). Match against member urls while the loser rows still exist; --- last_synced_at is the packages watermark, bumping it ships the correction --- to Tinybird. +-- same reason). Joined on repos directly, not the merge map, so it also +-- covers packages pointing at mixed-case singletons normalized below; runs +-- while loser urls still exist to match against. last_synced_at is the +-- packages watermark, bumping it ships the correction to Tinybird. UPDATE packages p SET repository_url = LOWER(p.repository_url), last_synced_at = NOW() -FROM repo_merge_members m -JOIN repos r ON r.id = m.repo_id -WHERE p.repository_url = r.url +FROM repos r +WHERE r.host = 'github' + AND p.repository_url = r.url AND p.repository_url <> LOWER(p.repository_url); -- Cascades security_contacts and repo_activity_snapshot on losers. @@ -86,14 +87,17 @@ USING repo_merge_members m WHERE r.id = m.repo_id AND NOT m.is_keeper; --- Groups that had no lowercase variant: lowercase the surviving row. Safe --- against UNIQUE (url) — every row sharing LOWER(url) was in the same group, --- and its losers are gone by now. +-- Lowercase every surviving mixed-case github url: keepers whose group had +-- no lowercase variant, plus mixed-case singletons that never had a +-- duplicate. Singletons must be normalized before the guard index exists — +-- writers upsert with ON CONFLICT (url), which does not arbitrate on the +-- LOWER(url) index, so a later insert of the same repo's canonical lowercase +-- url (all writers lowercase now) would raise unique_violation instead of +-- upserting. Safe against UNIQUE (url): any two rows sharing LOWER(url) were +-- a group above, and their losers are gone by now. UPDATE repos r SET url = LOWER(r.url), updated_at = NOW() -FROM repo_merge_members m -WHERE r.id = m.repo_id - AND m.is_keeper +WHERE r.host = 'github' AND r.url <> LOWER(r.url); -- The recurrence-guard unique index lives in the next migration: it must be diff --git a/backend/src/osspckgs/migrations/V1784718694__repos_github_lower_url_guard.sql b/backend/src/osspckgs/migrations/V1784718694__repos_github_lower_url_guard.sql index b99f542321..89c445cff3 100644 --- a/backend/src/osspckgs/migrations/V1784718694__repos_github_lower_url_guard.sql +++ b/backend/src/osspckgs/migrations/V1784718694__repos_github_lower_url_guard.sql @@ -13,7 +13,9 @@ -- -- A failed concurrent build leaves an INVALID index behind; the DROP lets a -- re-run replace it, where IF NOT EXISTS would silently keep the broken one. -DROP INDEX IF EXISTS repos_github_lower_url_uq; +-- Also CONCURRENTLY: a plain drop takes an ACCESS EXCLUSIVE lock on repos, +-- stalling the workers on the retry path this exists for. +DROP INDEX CONCURRENTLY IF EXISTS repos_github_lower_url_uq; CREATE UNIQUE INDEX CONCURRENTLY repos_github_lower_url_uq ON repos (LOWER(url)) From eff5795e8aa0324ae4f954c82040291c3554bc5f Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Wed, 22 Jul 2026 13:28:39 +0100 Subject: [PATCH 4/4] fix: code review Signed-off-by: Mouad BANI --- ...18693__merge_case_duplicate_github_repos.sql | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/backend/src/osspckgs/migrations/V1784718693__merge_case_duplicate_github_repos.sql b/backend/src/osspckgs/migrations/V1784718693__merge_case_duplicate_github_repos.sql index 6cab95471c..496166fc57 100644 --- a/backend/src/osspckgs/migrations/V1784718693__merge_case_duplicate_github_repos.sql +++ b/backend/src/osspckgs/migrations/V1784718693__merge_case_duplicate_github_repos.sql @@ -51,9 +51,20 @@ WHERE id IN ( WHERE rn > 1 ); -UPDATE package_repos pr -SET repo_id = m.keeper_id -FROM repo_merge_members m +-- Re-point surviving loser links via insert + delete, never an UPDATE: +-- repo_id is part of the Tinybird packageRepos sorting key (CDC models key +-- changes as delete + insert; a key-mutating UPDATE would leave both links +-- live under FINAL with no way to reconcile). The fresh row streams cleanly +-- with a new id and a NOW() version; the stale loser keys (these deletes +-- included) are removed by the post-migration Tinybird datasource rebuild. +INSERT INTO package_repos (package_id, repo_id, source, confidence, verified_at, created_at) +SELECT pr.package_id, m.keeper_id, pr.source, pr.confidence, NOW(), NOW() +FROM package_repos pr +JOIN repo_merge_members m ON m.repo_id = pr.repo_id +WHERE NOT m.is_keeper; + +DELETE FROM package_repos pr +USING repo_merge_members m WHERE pr.repo_id = m.repo_id AND NOT m.is_keeper;