fix: normalize and save repo url in rubygems (#4330)

Signed-off-by: Mouad BANI <mouad-mb@outlook.com>

Author: mbani01

services/apps/packages_worker/src/rubygems/normalize.ts

@@ -1,3 +1,5 @@
+import { canonicalizeRepoUrl } from '../utils/canonicalizeRepoUrl'
+
 import {
   NormalizedRubyGemsOwner,
   NormalizedRubyGemsPackage,
@@ -15,10 +17,12 @@ function nonEmpty(value: string | null | undefined): string | null {
 
 export function normalizeRubyGemsPackage(doc: RubyGemsGemResponse): NormalizedRubyGemsPackage {
   const licenses = doc.licenses && doc.licenses.length > 0 ? doc.licenses : null
+  const declaredRepositoryUrl = nonEmpty(doc.source_code_uri)
   return {
     description: nonEmpty(doc.info),
     homepage: nonEmpty(doc.homepage_uri),
-    declaredRepositoryUrl: nonEmpty(doc.source_code_uri),
+    declaredRepositoryUrl,
+    repo: declaredRepositoryUrl ? canonicalizeRepoUrl(declaredRepositoryUrl) : null,
     licenses,
     licensesRaw: licenses ? licenses.join(', ') : null,
     latestVersion: nonEmpty(doc.version),

services/apps/packages_worker/src/rubygems/runRubyGemsCoreLoop.ts

@@ -1,10 +1,12 @@
 import {
   QueryExecutor,
   RubyGemsPackageToSync,
+  getOrCreateRepoByUrl,
   listRubyGemsPackagesToSync,
   logAuditFieldChange,
   recordDownloadSnapshot,
   upsertPackage,
+  upsertPackageRepo,
 } from '@crowd/data-access-layer'
 import { getServiceChildLogger } from '@crowd/logging'
 
@@ -106,6 +108,7 @@ async function processPackage(
         description: normalized.description,
         homepage: normalized.homepage,
         declaredRepositoryUrl: normalized.declaredRepositoryUrl,
+        repositoryUrl: normalized.repo?.url ?? null,
         licenses: normalized.licenses,
         licensesRaw: normalized.licensesRaw,
         latestVersion: normalized.latestVersion,
@@ -114,6 +117,24 @@ async function processPackage(
       })
       pkgChanged.forEach((f) => changed.add(f))
 
+      if (normalized.repo) {
+        const { id: repoId, changedFields: repoChanged } = await getOrCreateRepoByUrl(
+          t,
+          normalized.repo.url,
+          normalized.repo.host,
+        )
+        repoChanged.forEach((f) => changed.add(f))
+
+        const linkChanged = await upsertPackageRepo(
+          t,
+          packageDbId.toString(),
+          repoId,
+          'declared',
+          0.8,
+        )
+        linkChanged.forEach((f) => changed.add(f))
+      }
+
       if (normalized.totalDownloads > 0) {
         const dlChanged = await recordDownloadSnapshot(t, {
           packageId: packageDbId,

services/apps/packages_worker/src/rubygems/types.ts

@@ -1,3 +1,5 @@
+import { CanonicalRepo } from '../utils/canonicalizeRepoUrl'
+
 export interface BatchResult {
   processed: number
   skipped: number
@@ -43,6 +45,7 @@ export interface NormalizedRubyGemsPackage {
   description: string | null
   homepage: string | null
   declaredRepositoryUrl: string | null
+  repo: CanonicalRepo | null
   licenses: string[] | null
   licensesRaw: string | null
   latestVersion: string | null