From 6cd406a64f05aa81206701e2615e24524a0ff557 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Mon, 13 Jul 2026 10:58:59 +0100 Subject: [PATCH 1/2] fix: normalize and store repository_url Signed-off-by: Mouad BANI --- services/apps/packages_worker/src/rubygems/normalize.ts | 6 +++++- .../packages_worker/src/rubygems/runRubyGemsCoreLoop.ts | 3 +++ services/apps/packages_worker/src/rubygems/types.ts | 1 + 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/services/apps/packages_worker/src/rubygems/normalize.ts b/services/apps/packages_worker/src/rubygems/normalize.ts index 7f8b651bf4..36bdb0e471 100644 --- a/services/apps/packages_worker/src/rubygems/normalize.ts +++ b/services/apps/packages_worker/src/rubygems/normalize.ts @@ -1,3 +1,5 @@ +import { normalizeScmUrl } from '../maven/extract' + import { NormalizedRubyGemsOwner, NormalizedRubyGemsPackage, @@ -15,10 +17,12 @@ function nonEmpty(value: string | null | undefined): string | null { export function normalizeRubyGemsPackage(doc: RubyGemsGemResponse): NormalizedRubyGemsPackage { const licenses = doc.licenses && doc.licenses.length > 0 ? doc.licenses : null + const declaredRepositoryUrl = nonEmpty(doc.source_code_uri) return { description: nonEmpty(doc.info), homepage: nonEmpty(doc.homepage_uri), - declaredRepositoryUrl: nonEmpty(doc.source_code_uri), + declaredRepositoryUrl, + repositoryUrl: normalizeScmUrl(declaredRepositoryUrl), licenses, licensesRaw: licenses ? licenses.join(', ') : null, latestVersion: nonEmpty(doc.version), diff --git a/services/apps/packages_worker/src/rubygems/runRubyGemsCoreLoop.ts b/services/apps/packages_worker/src/rubygems/runRubyGemsCoreLoop.ts index 775c2d6108..f14bab0ac6 100644 --- a/services/apps/packages_worker/src/rubygems/runRubyGemsCoreLoop.ts +++ b/services/apps/packages_worker/src/rubygems/runRubyGemsCoreLoop.ts @@ -45,6 +45,7 @@ async function markPackageError(qx: QueryExecutor, pkg: RubyGemsPackageToSync): description: null, homepage: null, declaredRepositoryUrl: null, + repositoryUrl: null, licenses: null, licensesRaw: null, latestVersion: pkg.latestVersion, @@ -73,6 +74,7 @@ async function processPackage( description: null, homepage: null, declaredRepositoryUrl: null, + repositoryUrl: null, licenses: null, licensesRaw: null, latestVersion: pkg.latestVersion, @@ -106,6 +108,7 @@ async function processPackage( description: normalized.description, homepage: normalized.homepage, declaredRepositoryUrl: normalized.declaredRepositoryUrl, + repositoryUrl: normalized.repositoryUrl, licenses: normalized.licenses, licensesRaw: normalized.licensesRaw, latestVersion: normalized.latestVersion, diff --git a/services/apps/packages_worker/src/rubygems/types.ts b/services/apps/packages_worker/src/rubygems/types.ts index f1948ce1d1..fd78d99451 100644 --- a/services/apps/packages_worker/src/rubygems/types.ts +++ b/services/apps/packages_worker/src/rubygems/types.ts @@ -43,6 +43,7 @@ export interface NormalizedRubyGemsPackage { description: string | null homepage: string | null declaredRepositoryUrl: string | null + repositoryUrl: string | null licenses: string[] | null licensesRaw: string | null latestVersion: string | null From 44f3b19b0dc2b376c20c37647733ee4376d1f899 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Mon, 13 Jul 2026 11:10:11 +0100 Subject: [PATCH 2/2] fix: re-use canonicalizeRepoUrl & insert package_repos Signed-off-by: Mouad BANI --- .../packages_worker/src/rubygems/normalize.ts | 4 ++-- .../src/rubygems/runRubyGemsCoreLoop.ts | 24 ++++++++++++++++--- .../packages_worker/src/rubygems/types.ts | 4 +++- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/services/apps/packages_worker/src/rubygems/normalize.ts b/services/apps/packages_worker/src/rubygems/normalize.ts index 36bdb0e471..cae2dcaac2 100644 --- a/services/apps/packages_worker/src/rubygems/normalize.ts +++ b/services/apps/packages_worker/src/rubygems/normalize.ts @@ -1,4 +1,4 @@ -import { normalizeScmUrl } from '../maven/extract' +import { canonicalizeRepoUrl } from '../utils/canonicalizeRepoUrl' import { NormalizedRubyGemsOwner, @@ -22,7 +22,7 @@ export function normalizeRubyGemsPackage(doc: RubyGemsGemResponse): NormalizedRu description: nonEmpty(doc.info), homepage: nonEmpty(doc.homepage_uri), declaredRepositoryUrl, - repositoryUrl: normalizeScmUrl(declaredRepositoryUrl), + repo: declaredRepositoryUrl ? canonicalizeRepoUrl(declaredRepositoryUrl) : null, licenses, licensesRaw: licenses ? licenses.join(', ') : null, latestVersion: nonEmpty(doc.version), diff --git a/services/apps/packages_worker/src/rubygems/runRubyGemsCoreLoop.ts b/services/apps/packages_worker/src/rubygems/runRubyGemsCoreLoop.ts index f14bab0ac6..455b582eda 100644 --- a/services/apps/packages_worker/src/rubygems/runRubyGemsCoreLoop.ts +++ b/services/apps/packages_worker/src/rubygems/runRubyGemsCoreLoop.ts @@ -1,10 +1,12 @@ import { QueryExecutor, RubyGemsPackageToSync, + getOrCreateRepoByUrl, listRubyGemsPackagesToSync, logAuditFieldChange, recordDownloadSnapshot, upsertPackage, + upsertPackageRepo, } from '@crowd/data-access-layer' import { getServiceChildLogger } from '@crowd/logging' @@ -45,7 +47,6 @@ async function markPackageError(qx: QueryExecutor, pkg: RubyGemsPackageToSync): description: null, homepage: null, declaredRepositoryUrl: null, - repositoryUrl: null, licenses: null, licensesRaw: null, latestVersion: pkg.latestVersion, @@ -74,7 +75,6 @@ async function processPackage( description: null, homepage: null, declaredRepositoryUrl: null, - repositoryUrl: null, licenses: null, licensesRaw: null, latestVersion: pkg.latestVersion, @@ -108,7 +108,7 @@ async function processPackage( description: normalized.description, homepage: normalized.homepage, declaredRepositoryUrl: normalized.declaredRepositoryUrl, - repositoryUrl: normalized.repositoryUrl, + repositoryUrl: normalized.repo?.url ?? null, licenses: normalized.licenses, licensesRaw: normalized.licensesRaw, latestVersion: normalized.latestVersion, @@ -117,6 +117,24 @@ async function processPackage( }) pkgChanged.forEach((f) => changed.add(f)) + if (normalized.repo) { + const { id: repoId, changedFields: repoChanged } = await getOrCreateRepoByUrl( + t, + normalized.repo.url, + normalized.repo.host, + ) + repoChanged.forEach((f) => changed.add(f)) + + const linkChanged = await upsertPackageRepo( + t, + packageDbId.toString(), + repoId, + 'declared', + 0.8, + ) + linkChanged.forEach((f) => changed.add(f)) + } + if (normalized.totalDownloads > 0) { const dlChanged = await recordDownloadSnapshot(t, { packageId: packageDbId, diff --git a/services/apps/packages_worker/src/rubygems/types.ts b/services/apps/packages_worker/src/rubygems/types.ts index fd78d99451..3d65543f51 100644 --- a/services/apps/packages_worker/src/rubygems/types.ts +++ b/services/apps/packages_worker/src/rubygems/types.ts @@ -1,3 +1,5 @@ +import { CanonicalRepo } from '../utils/canonicalizeRepoUrl' + export interface BatchResult { processed: number skipped: number @@ -43,7 +45,7 @@ export interface NormalizedRubyGemsPackage { description: string | null homepage: string | null declaredRepositoryUrl: string | null - repositoryUrl: string | null + repo: CanonicalRepo | null licenses: string[] | null licensesRaw: string | null latestVersion: string | null