Feat/track packages (#4169)

Signed-off-by: Uroš Marolt <uros@marolt.me>

Author: themarolt

services/apps/packages_worker/src/deps-dev/canonicalRepoUrl.ts

@@ -1,12 +1,12 @@
-const HOST_BY_TYPE: Record<string, string> = {
+const DOMAIN_BY_TYPE: Record<string, string> = {
   GITHUB: 'github.com',
   GITLAB: 'gitlab.com',
   BITBUCKET: 'bitbucket.org',
 }
 
 export function canonicalRepoUrl(type: string, name: string): string | null {
-  const host = HOST_BY_TYPE[type]
-  if (!host) return null
+  const domain = DOMAIN_BY_TYPE[type]
+  if (!domain) return null
   // Names are bare paths (owner/repo or owner/subgroup/repo) — no host prefix.
   // Strip full URL prefix only if present; bare paths pass through untouched.
   // TODO(Step 2a): verify against sampled ProjectName shapes from BQ console.
@@ -18,16 +18,22 @@ export function canonicalRepoUrl(type: string, name: string): string | null {
   // GitHub enforces case-insensitive uniqueness (can't create Foo and foo as separate orgs/repos),
   // so lowercasing is always safe and aligns with deps.dev's canonical form.
   // GitLab, Bitbucket, and self-hosted forges are case-sensitive — preserve original casing.
-  const path = host === 'github.com' ? stripped.toLowerCase() : stripped
+  const path = type === 'GITHUB' ? stripped.toLowerCase() : stripped
   if (!/^[a-zA-Z0-9._-]+\/[a-zA-Z0-9._/-]+$/.test(path)) return null
-  return `https://${host}/${path}`
+  return `https://${domain}/${path}`
+}
+
+const HOST_BY_DOMAIN: Record<string, string> = {
+  'github.com': 'github',
+  'gitlab.com': 'gitlab',
+  'bitbucket.org': 'bitbucket',
 }
 
 export function parseRepoUrl(url: string): { host: string; owner: string; name: string } {
   const u = new URL(url)
   const parts = u.pathname.slice(1).split('/')
   return {
-    host: u.hostname,
+    host: HOST_BY_DOMAIN[u.hostname] ?? u.hostname,
     owner: parts[0],
     name: parts.slice(1).join('/'),
   }

services/apps/packages_worker/src/deps-dev/queries/packageReposSql.ts

@@ -11,9 +11,9 @@ path_computed AS (
   SELECT
     pm.purl,
     CASE pvp.ProjectType
-      WHEN 'GITHUB'    THEN 'github.com'
-      WHEN 'GITLAB'    THEN 'gitlab.com'
-      WHEN 'BITBUCKET' THEN 'bitbucket.org'
+      WHEN 'GITHUB'    THEN 'github'
+      WHEN 'GITLAB'    THEN 'gitlab'
+      WHEN 'BITBUCKET' THEN 'bitbucket'
     END AS host,
     CASE pvp.ProjectType
       WHEN 'GITHUB' THEN LOWER(
@@ -40,7 +40,7 @@ path_computed AS (
 )
 SELECT
   purl,
-  CONCAT('https://', host, '/', path) AS canonical_url,
+  CONCAT('https://', CASE host WHEN 'github' THEN 'github.com' WHEN 'gitlab' THEN 'gitlab.com' WHEN 'bitbucket' THEN 'bitbucket.org' END, '/', path) AS canonical_url,
   confidence
 FROM path_computed
 WHERE REGEXP_CONTAINS(path, r'^[a-zA-Z0-9._-]+/[a-zA-Z0-9._/-]+$')

services/apps/packages_worker/src/deps-dev/queries/reposSql.ts

@@ -5,9 +5,9 @@ WITH raw AS (
     p.Type AS raw_project_type,
     p.Name AS raw_project_name,
     CASE p.Type
-      WHEN 'GITHUB'    THEN 'github.com'
-      WHEN 'GITLAB'    THEN 'gitlab.com'
-      WHEN 'BITBUCKET' THEN 'bitbucket.org'
+      WHEN 'GITHUB'    THEN 'github'
+      WHEN 'GITLAB'    THEN 'gitlab'
+      WHEN 'BITBUCKET' THEN 'bitbucket'
     END AS host,
     CASE p.Type
       WHEN 'GITHUB' THEN LOWER(
@@ -32,7 +32,7 @@ WITH raw AS (
     AND p.Type IN ('GITHUB', 'GITLAB', 'BITBUCKET')
 )
 SELECT
-  CONCAT('https://', host, '/', path) AS canonical_url,
+  CONCAT('https://', CASE raw_project_type WHEN 'GITHUB' THEN 'github.com' WHEN 'GITLAB' THEN 'gitlab.com' WHEN 'BITBUCKET' THEN 'bitbucket.org' END, '/', path) AS canonical_url,
   host,
   REGEXP_EXTRACT(path, r'^([^/]+)/') AS owner,
   REGEXP_EXTRACT(path, r'^[^/]+/(.+)$') AS name,