fix: get repositories using xml endpoint

Signed-off-by: Mouad BANI <mouad-mb@outlook.com>

Author: mbani01

services/apps/packages_worker/src/nuget/client.ts

@@ -8,6 +8,7 @@ import {
 } from './types'
 
 const SERVICE_INDEX_URL = 'https://api.nuget.org/v3/index.json'
+const FLAT_CONTAINER_BASE_URL = 'https://api.nuget.org/v3-flatcontainer'
 
 interface ServiceIndexResource {
   '@id': string
@@ -144,3 +145,29 @@ export async function fetchRegistration(
     throw err
   }
 }
+
+// The registration API never exposes the nuspec <repository> element — it must be read from the
+// raw nuspec XML, served by the flat-container endpoint.
+export async function fetchNuspec(
+  packageId: string,
+  version: string,
+): Promise<string | NuGetFetchError> {
+  const lowerId = packageId.toLowerCase()
+  const lowerVersion = version.toLowerCase()
+
+  try {
+    const resp = await axios.get<string>(
+      `${FLAT_CONTAINER_BASE_URL}/${lowerId}/${lowerVersion}/${lowerId}.nuspec`,
+      {
+        responseType: 'text',
+        transformResponse: (data) => data,
+        timeout: 15000,
+      },
+    )
+    return resp.data
+  } catch (err) {
+    const classified = classifyError(err)
+    if (classified) return classified
+    throw err
+  }
+}

services/apps/packages_worker/src/nuget/normalize.ts

@@ -1,3 +1,5 @@
+import { XMLParser } from 'fast-xml-parser'
+
 import { canonicalizeRepoUrl } from '../utils/canonicalizeRepoUrl'
 
 import {
@@ -28,6 +30,18 @@ function parsePublishedDate(published: string | undefined): Date | null {
   return !isNaN(date.getTime()) && date.getUTCFullYear() > 1900 ? date : null
 }
 
+const nuspecParser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: '@_' })
+
+export function parseNuspecRepositoryUrl(nuspecXml: string): string | null {
+  try {
+    const doc = nuspecParser.parse(nuspecXml)
+    const url = doc?.package?.metadata?.repository?.['@_url']
+    return typeof url === 'string' && url.trim() !== '' ? url.trim() : null
+  } catch {
+    return null
+  }
+}
+
 const SCM_HOSTS = ['github.com', 'gitlab.com', 'bitbucket.org']
 
 function isScmUrl(url: string | undefined): boolean {
@@ -56,6 +70,7 @@ export function normalizeNuGetPackage(
   packageId: string,
   searchResult: NuGetSearchItem | null,
   registration: NuGetRegistrationIndex,
+  nuspecXml?: string | null,
 ): NormalizedNuGetPackage {
   const allLeaves = registration.items.flatMap((page) => page.items ?? [])
   const allEntries: NuGetCatalogEntry[] = allLeaves.map((leaf) => leaf.catalogEntry)
@@ -88,7 +103,9 @@ export function normalizeNuGetPackage(
         ...(latestEntry ? [latestEntry] : []),
       ]
     : [...allEntries].reverse()
-  const nuspecRepoUrl = entriesForRepo.find((e) => e.repository?.url)?.repository?.url
+  const catalogRepoUrl = entriesForRepo.find((e) => e.repository?.url)?.repository?.url
+  const fetchedNuspecRepoUrl = nuspecXml ? parseNuspecRepositoryUrl(nuspecXml) : null
+  const nuspecRepoUrl = fetchedNuspecRepoUrl ?? catalogRepoUrl
   const declaredRepositoryUrl = nuspecRepoUrl ?? null
   const repo =
     (nuspecRepoUrl ? canonicalizeRepoUrl(nuspecRepoUrl) : null) ??

services/apps/packages_worker/src/nuget/runNuGetEnrichmentLoop.ts

@@ -17,7 +17,7 @@ import { getServiceChildLogger } from '@crowd/logging'
 
 import { getNuGetConfig } from '../config'
 
-import { fetchRegistration, fetchSearch } from './client'
+import { fetchNuspec, fetchRegistration, fetchSearch } from './client'
 import { normalizeNuGetPackage } from './normalize'
 import { BatchResult, isNuGetFetchError } from './types'
 
@@ -98,7 +98,15 @@ async function processPackage(
 
   const searchItem = isNuGetFetchError(searchResult) ? null : searchResult
 
-  const normalized = normalizeNuGetPackage(packageId, searchItem, registrationResult)
+  const preliminary = normalizeNuGetPackage(packageId, searchItem, registrationResult)
+
+  let nuspecXml: string | null = null
+  if (preliminary.latestVersion) {
+    const nuspecResult = await fetchNuspec(packageId, preliminary.latestVersion)
+    nuspecXml = isNuGetFetchError(nuspecResult) ? null : nuspecResult
+  }
+
+  const normalized = normalizeNuGetPackage(packageId, searchItem, registrationResult, nuspecXml)
 
   await withDeadlockRetry(() =>
     qx.tx(async (t) => {