fix: fixes after prod test

Signed-off-by: Mouad BANI <mouad-mb@outlook.com>

Author: mbani01

services/apps/packages_worker/package.json

@@ -23,6 +23,8 @@
     "trigger-bootstrap:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=bq-dataset-ingest tsx src/scripts/triggerBootstrap.ts",
     "trigger-go": "SERVICE=go-worker tsx src/scripts/triggerGoEnrich.ts",
     "trigger-go:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=go-worker tsx src/scripts/triggerGoEnrich.ts",
+    "trigger-nuget": "SERVICE=nuget-worker tsx src/scripts/triggerNuGetSync.ts",
+    "trigger-nuget:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=nuget-worker tsx src/scripts/triggerNuGetSync.ts",
     "start:npm-worker": "CROWD_TEMPORAL_TASKQUEUE=npm-worker SERVICE=npm-worker tsx src/bin/npm-worker.ts",
     "dev:npm-worker": "CROWD_TEMPORAL_TASKQUEUE=npm-worker SERVICE=npm-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9236 src/bin/npm-worker.ts",
     "dev:npm-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=npm-worker SERVICE=npm-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9236 src/bin/npm-worker.ts",

services/apps/packages_worker/src/nuget/client.ts

@@ -65,9 +65,7 @@ function classifyError(err: unknown): NuGetFetchError | null {
   return null
 }
 
-export async function fetchSearch(
-  packageId: string,
-): Promise<NuGetSearchItem | NuGetFetchError> {
+export async function fetchSearch(packageId: string): Promise<NuGetSearchItem | NuGetFetchError> {
   const { searchBaseUrl } = await resolveEndpoints()
   const lowerPackageId = packageId.toLowerCase()
 
@@ -97,12 +95,21 @@ export async function fetchSearch(
   }
 }
 
-async function fetchRegistrationPage(pageId: string): Promise<NuGetRegistrationPage> {
-  const resp = await axios.get<NuGetRegistrationPage>(pageId, {
-    headers: { 'Accept-Encoding': 'gzip' },
-    timeout: 15000,
-  })
-  return resp.data
+async function fetchRegistrationPage(
+  pageId: string,
+  maxAttempts = 2,
+): Promise<NuGetRegistrationPage> {
+  for (let attempt = 1; ; attempt++) {
+    try {
+      const resp = await axios.get<NuGetRegistrationPage>(pageId, {
+        headers: { 'Accept-Encoding': 'gzip' },
+        timeout: 15000,
+      })
+      return resp.data
+    } catch (err) {
+      if (attempt >= maxAttempts) throw err
+    }
+  }
 }
 
 export async function fetchRegistration(
@@ -125,12 +132,8 @@ export async function fetchRegistration(
     for (let i = 0; i < index.items.length; i++) {
       const page = index.items[i]
       if (!page.items) {
-        try {
-          const fullPage = await fetchRegistrationPage(page['@id'])
-          index.items[i] = { ...page, items: fullPage.items ?? [] }
-        } catch {
-          index.items[i] = { ...page, items: [] }
-        }
+        const fullPage = await fetchRegistrationPage(page['@id'])
+        index.items[i] = { ...page, items: fullPage.items ?? [] }
       }
     }
 

services/apps/packages_worker/src/nuget/normalize.ts

@@ -19,6 +19,8 @@ function isPrerelease(version: string): boolean {
   return version.includes('-')
 }
 
+const SCM_HOSTS = ['github.com', 'gitlab.com', 'bitbucket.org']
+
 function normalizeRepoUrl(url: string | undefined): string | null {
   if (!url) return null
   return url
@@ -29,6 +31,15 @@ function normalizeRepoUrl(url: string | undefined): string | null {
     .replace(/^http:\/\/github\.com\//, 'https://github.com/')
 }
 
+function isScmUrl(url: string | undefined): boolean {
+  if (!url) return false
+  try {
+    return SCM_HOSTS.some((h) => new URL(url).hostname.endsWith(h))
+  } catch {
+    return false
+  }
+}
+
 function parseLicense(
   licenseExpression: string | undefined,
   licenseUrl: string | undefined,
@@ -69,10 +80,19 @@ export function normalizeNuGetPackage(
 
   const homepage = searchResult?.projectUrl || latestListedEntry?.projectUrl || null
 
-  const declaredRepositoryUrl = latestEntry4License?.repository?.url
-    ? latestEntry4License.repository.url
-    : null
-  const repositoryUrl = normalizeRepoUrl(declaredRepositoryUrl)
+  // Scan all entries (prefer latest listed, then any) for a nuspec <repository> url.
+  // Fall back to a SCM-shaped projectUrl/homepage when no nuspec repository is present.
+  const entriesForRepo = latestListedEntry
+    ? [
+        latestListedEntry,
+        ...listedEntries.slice(0, -1).reverse(),
+        ...(latestEntry ? [latestEntry] : []),
+      ]
+    : [...allEntries].reverse()
+  const nuspecRepoUrl = entriesForRepo.find((e) => e.repository?.url)?.repository?.url
+  const declaredRepositoryUrl = nuspecRepoUrl ?? null
+  const repoCandidate = nuspecRepoUrl ?? (isScmUrl(homepage) ? homepage : null)
+  const repositoryUrl = normalizeRepoUrl(repoCandidate ?? undefined)
 
   const keywords = searchResult?.tags && searchResult.tags.length > 0 ? searchResult.tags : null
 
@@ -85,16 +105,25 @@ export function normalizeNuGetPackage(
     status = 'active'
   }
 
+  // NuGet stamps unlisted versions with 1900-01-01T00:00:00Z as a sentinel — exclude them.
   const publishedDates = allEntries
     .filter((e) => e.published)
     .map((e) => new Date(e.published as string))
-    .filter((d) => !isNaN(d.getTime()))
+    .filter((d) => !isNaN(d.getTime()) && d.getUTCFullYear() > 1900)
     .sort((a, b) => a.getTime() - b.getTime())
 
   const firstReleaseAt = publishedDates.length > 0 ? publishedDates[0] : null
 
   const latestEntry4Date = latestListedEntry ?? latestEntry
-  const latestReleaseAt = latestEntry4Date?.published ? new Date(latestEntry4Date.published) : null
+  const latestReleaseAtRaw = latestEntry4Date?.published
+    ? new Date(latestEntry4Date.published)
+    : null
+  const latestReleaseAt =
+    latestReleaseAtRaw &&
+    !isNaN(latestReleaseAtRaw.getTime()) &&
+    latestReleaseAtRaw.getUTCFullYear() > 1900
+      ? latestReleaseAtRaw
+      : null
 
   const totalDownloads = searchResult?.totalDownloads ?? 0
 
@@ -136,7 +165,7 @@ export function normalizeNuGetPackage(
     latestVersion,
     versionsCount: allEntries.length,
     firstReleaseAt,
-    latestReleaseAt: latestReleaseAt && !isNaN(latestReleaseAt.getTime()) ? latestReleaseAt : null,
+    latestReleaseAt,
     totalDownloads,
     owners,
     authors,

services/apps/packages_worker/src/nuget/runNuGetEnrichmentLoop.ts

@@ -4,6 +4,7 @@ import {
   QueryExecutor,
   listNuGetPackagesToSync,
   logAuditFieldChange,
+  markNuGetPackageError,
   recordNuGetDownloadSnapshot,
   replacePackageMaintainers,
   upsertMaintainer,
@@ -79,14 +80,13 @@ async function processPackage(
         latestReleaseAt: null,
         registryUrl: nugetRegistryUrl(packageId),
         ingestionSource: 'nuget_not_found',
-        dependentPackagesCount: pkg.dependentPackagesCount,
-        dependentReposCount: pkg.dependentReposCount,
       })
       log.warn({ purl: pkg.purl }, 'Package not found on NuGet registry — writing minimal record')
       return 'skipped'
     }
     if (registrationResult.kind === 'RATE_LIMIT') {
       log.warn({ purl: pkg.purl }, 'Rate limited by NuGet registry — will retry next pass')
+      await markNuGetPackageError(qx, pkg.purl)
       return 'error'
     }
     throw new Error(
@@ -119,8 +119,6 @@ async function processPackage(
         latestReleaseAt: normalized.latestReleaseAt,
         registryUrl: nugetRegistryUrl(packageId),
         ingestionSource: 'nuget-registry',
-        dependentPackagesCount: pkg.dependentPackagesCount,
-        dependentReposCount: pkg.dependentReposCount,
       })
       pkgChanged.forEach((f) => changed.add(f))
 
@@ -224,6 +222,11 @@ export async function processBatch(
         } catch (err) {
           const message = err instanceof Error ? err.message : String(err)
           log.error({ purl: pkg.purl, error: message }, 'Unexpected error processing package')
+          try {
+            await markNuGetPackageError(qx, pkg.purl)
+          } catch {
+            // best-effort — don't let a failed mark crash the batch
+          }
           counts.error++
         }
       }),

services/libs/data-access-layer/src/osspckgs/nuget.ts

@@ -8,8 +8,6 @@ export type NuGetPackageToSync = {
   id: number
   purl: string
   name: string
-  dependentPackagesCount: number | null
-  dependentReposCount: number | null
   latestVersion: string | null
 }
 
@@ -30,8 +28,6 @@ export type IDbNuGetPackageUpsert = {
   latestReleaseAt: Date | null
   registryUrl: string | null
   ingestionSource: string
-  dependentPackagesCount: number | null
-  dependentReposCount: number | null
 }
 
 export type IDbNuGetVersionUpsert = {
@@ -67,21 +63,22 @@ export async function upsertNuGetPackage(
         description, homepage, declared_repository_url, repository_url,
         licenses, licenses_raw, keywords, status,
         latest_version, versions_count, first_release_at, latest_release_at,
-        registry_url, ingestion_source, dependent_count, dependent_repos_count,
+        registry_url, ingestion_source,
         last_synced_at, created_at
       ) VALUES (
         $(purl), 'nuget', NULL, $(name),
         $(description), $(homepage), $(declaredRepositoryUrl), $(repositoryUrl),
         $(licenses)::text[], $(licensesRaw), $(keywords)::text[], $(status),
         $(latestVersion), $(versionsCount), $(firstReleaseAt), $(latestReleaseAt),
-        $(registryUrl), $(ingestionSource), $(dependentPackagesCount), $(dependentReposCount),
+        $(registryUrl), $(ingestionSource),
         NOW(), NOW()
       )
       ON CONFLICT (purl) DO UPDATE SET
         description              = COALESCE(EXCLUDED.description,              packages.description),
         homepage                 = COALESCE(EXCLUDED.homepage,                 packages.homepage),
         declared_repository_url  = COALESCE(EXCLUDED.declared_repository_url,  packages.declared_repository_url),
-        repository_url           = COALESCE(EXCLUDED.repository_url,           packages.repository_url),
+        repository_url           = COALESCE(EXCLUDED.repository_url,           packages.repository_url,
+                                            packages.declared_repository_url),
         licenses                 = COALESCE(EXCLUDED.licenses,                 packages.licenses),
         licenses_raw             = COALESCE(EXCLUDED.licenses_raw,             packages.licenses_raw),
         keywords                 = COALESCE(EXCLUDED.keywords,                 packages.keywords),
@@ -92,8 +89,6 @@ export async function upsertNuGetPackage(
         latest_release_at        = COALESCE(EXCLUDED.latest_release_at,        packages.latest_release_at),
         registry_url             = COALESCE(EXCLUDED.registry_url,             packages.registry_url),
         ingestion_source         = EXCLUDED.ingestion_source,
-        dependent_count          = COALESCE(EXCLUDED.dependent_count,          packages.dependent_count),
-        dependent_repos_count    = COALESCE(EXCLUDED.dependent_repos_count,    packages.dependent_repos_count),
         last_synced_at           = NOW()
       RETURNING id, description, homepage, declared_repository_url, repository_url,
                 licenses, licenses_raw, keywords, status,
@@ -136,8 +131,6 @@ export async function upsertNuGetPackage(
       latestReleaseAt: item.latestReleaseAt ?? null,
       registryUrl: item.registryUrl ?? null,
       ingestionSource: item.ingestionSource,
-      dependentPackagesCount: item.dependentPackagesCount ?? null,
-      dependentReposCount: item.dependentReposCount ?? null,
     },
   )
   return { id: row.id as number, changedFields: row.changed_fields as string[] }
@@ -246,9 +239,7 @@ export async function listNuGetPackagesToSync(
       p.id,
       p.purl,
       p.name,
-      p.dependent_count        AS "dependentPackagesCount",
-      p.dependent_repos_count  AS "dependentReposCount",
-      p.latest_version         AS "latestVersion"
+      p.latest_version AS "latestVersion"
     FROM packages p
     WHERE
       p.ecosystem = 'nuget'
@@ -267,6 +258,16 @@ export async function listNuGetPackagesToSync(
   )
 }
 
+// ─── Error outcome ────────────────────────────────────────────────────────────
+
+export async function markNuGetPackageError(qx: QueryExecutor, purl: string): Promise<void> {
+  await qx.result(
+    `UPDATE packages SET ingestion_source = 'nuget_error', last_synced_at = NOW()
+      WHERE purl = $(purl)`,
+    { purl },
+  )
+}
+
 // ─── Download snapshot ────────────────────────────────────────────────────────
 
 export async function recordNuGetDownloadSnapshot(