Merge branch 'main' into fix/revert-segment-middleware-leaf-expansion

Author: skwowet

backend/.env.dist.local

@@ -195,7 +195,7 @@ OSSPCKGS_GCP_CREDENTIALS_B64=e30=
 # maven/all.zip 404s). The allowlist check and DB storage normalize to lowercase
 # internally per ADR-0001 §OSV "Ecosystem normalization", so downstream stays lowercase.
 OSV_BULK_BASE_URL=https://osv-vulnerabilities.storage.googleapis.com
-OSV_ECOSYSTEMS=npm,Maven
+OSV_ECOSYSTEMS=npm,Maven,cargo
 OSV_TMP_DIR=/tmp/osv
 OSV_BATCH_SIZE=500
 OSV_DERIVE_BATCH_SIZE=1000

services/apps/packages_worker/src/npm/activities.ts

@@ -124,18 +124,23 @@ async function ingestOne(qx: QueryExecutor, purl: string, dispatcher?: Dispatche
     }
 
     // 429 / 5xx / network → bubble up so Temporal retries the activity with exponential backoff.
-    if (!isClientError(packumentResult.statusCode, packumentResult.kind)) {
+    // MALFORMED is permanent (a 200 body that isn't a packument — retrying won't change it),
+    // so it takes the quick-retry-then-skip path below instead of poisoning the lane forever.
+    if (
+      !isClientError(packumentResult.statusCode, packumentResult.kind) &&
+      packumentResult.kind !== 'MALFORMED'
+    ) {
       throw new Error(`Failed to fetch packument for ${name}: ${packumentResult.message}`)
     }
 
-    // 4xx → quick retry a few times (1s, 2s); give up and skip after the last attempt.
+    // 4xx/MALFORMED → quick retry a few times (1s, 2s); give up and skip after the last attempt.
     if (attempt < INGEST_4XX_ATTEMPTS) {
       await sleep(attempt * INGEST_4XX_BACKOFF_MS)
       continue
     }
     log.warn(
       { purl, statusCode: packumentResult.statusCode, kind: packumentResult.kind },
-      'npm packument 4xx after fast retries — marking scanned and skipping',
+      'npm packument 4xx/malformed after fast retries — marking scanned and skipping',
     )
     await markNpmPackageScanned(qx, purl, {
       status: 'error',

services/apps/packages_worker/src/npm/fetchPackument.ts

@@ -48,11 +48,34 @@ export async function fetchPackument(
     return { kind: 'MALFORMED', message: 'invalid JSON' }
   }
 
-  if (!isPackument(json)) return { kind: 'MALFORMED', message: 'unexpected shape' }
+  if (!isPackument(json)) {
+    const stub = asUnpublishedStub(json)
+    if (stub) return stub
+    return { kind: 'MALFORMED', message: 'unexpected shape' }
+  }
   delete (json as unknown as Record<string, unknown>).readme
   return json
 }
 
 function isPackument(v: unknown): v is Packument {
   return typeof v === 'object' && v !== null && 'name' in v && 'versions' in v && 'dist-tags' in v
 }
+
+// A fully unpublished package returns HTTP 200 with a stub document — just name + time,
+// where time.unpublished records the unpublish event; there are no versions/dist-tags keys,
+// so isPackument rejects it. Normalize the stub into an empty packument with `unpublished`
+// set, so ingest stores status='unpublished' instead of erroring on shape.
+function asUnpublishedStub(v: unknown): Packument | null {
+  if (typeof v !== 'object' || v === null) return null
+  const o = v as Record<string, unknown>
+  if (typeof o.name !== 'string' || typeof o.time !== 'object' || o.time === null) return null
+  const t = o.time as Record<string, unknown>
+  const unpublished = t.unpublished
+  if (typeof unpublished !== 'object' || unpublished === null) return null
+  if (typeof (unpublished as Record<string, unknown>).time !== 'string') return null
+  const time: Record<string, string> = {}
+  for (const [key, value] of Object.entries(t)) {
+    if (typeof value === 'string') time[key] = value
+  }
+  return { name: o.name, 'dist-tags': {}, versions: {}, time, unpublished }
+}

services/apps/packages_worker/src/npm/normalize.ts

@@ -56,6 +56,33 @@ export function normalizeLicenses(packument: Packument): string[] {
   )
 }
 
+// A version's `license` field comes in several shapes: a plain SPDX string ("MIT"),
+// an object ({ type, url }), or the legacy array form ([{ type, file }, ...]). Passing those
+// raw into a text column would persist objects/arrays, so collapse every shape to a single
+// string (OR-joined for the array form) or null. Non-string or blank `type` values are dropped.
+export function versionLicense(raw: unknown): string | null {
+  if (raw == null) return null
+  if (typeof raw === 'string') return blankToNull(raw)
+  if (Array.isArray(raw)) {
+    const types = raw
+      .map((l) => (typeof l === 'string' ? blankToNull(l) : licenseType(l)))
+      .filter((t): t is string => t !== null)
+    return types.length ? types.join(' OR ') : null
+  }
+  return licenseType(raw)
+}
+
+function licenseType(v: unknown): string | null {
+  if (typeof v !== 'object' || v === null) return null
+  const type = (v as { type?: unknown }).type
+  return typeof type === 'string' ? blankToNull(type) : null
+}
+
+function blankToNull(s: string): string | null {
+  const trimmed = s.trim()
+  return trimmed || null
+}
+
 function clean(s: string): string {
   return s.replace(/[()]/g, '').trim()
 }

services/apps/packages_worker/src/npm/upsertPackage.ts

@@ -15,6 +15,7 @@ import {
   normalizeLicenses,
   parseNpmName,
   stripNullBytesDeep,
+  versionLicense,
 } from './normalize'
 import type { FundingEntry, Packument } from './types'
 
@@ -98,7 +99,7 @@ export async function upsertPackage(
         publishedAt: time[number] ?? null,
         isLatest: number === latestVersion,
         isPrerelease: isPrerelease(number),
-        license: v.license ?? licenses[0] ?? null,
+        license: versionLicense(v.license) ?? licenses[0] ?? null,
       })),
     )
     verChanged.forEach((f) => changed.add(f))

services/apps/packages_worker/src/npm/workflows.ts

@@ -27,7 +27,7 @@ const heartbeatingActs = proxyActivities<typeof activities>({
 
 // Per lane, per round. Total purls fetched per round = lanes × INGEST_PER_LANE.
 const INGEST_PER_LANE = 50
-const INGEST_ROUNDS_PER_RUN = 25
+const INGEST_ROUNDS_PER_RUN = 5
 
 interface IngestState {
   unscannedCursor: string

services/apps/packages_worker/src/osv/__tests__/versionCompare.test.ts

@@ -6,7 +6,7 @@ import { compareVersion } from '../versionCompare'
 // the contract is -1/0/1, but we only care about the sign.
 const sign = (n: number | null) => (n === null ? null : Math.sign(n))
 
-describe('compareVersion — npm (semver)', () => {
+describe('compareVersion — npm/cargo (semver)', () => {
   it.each([
     ['1.2.3', '1.2.4', -1],
     ['1.2.4', '1.2.3', 1],
@@ -20,11 +20,21 @@ describe('compareVersion — npm (semver)', () => {
     expect(sign(compareVersion('npm', a, b))).toBe(expected)
   })
 
-  it('returns null for unparseable npm versions', () => {
+  it.each([
+    ['0.1.0', '0.2.0', -1],
+    ['1.0.0', '1.0.0', 0],
+    ['2.0.0', '1.9.9', 1],
+    ['0.3.0-alpha.1', '0.3.0', -1],
+  ])('compareVersion("cargo", %s, %s) sign = %s', (a, b, expected) => {
+    expect(sign(compareVersion('cargo', a, b))).toBe(expected)
+  })
+
+  it('returns null for unparseable semver versions', () => {
     expect(compareVersion('npm', 'not-a-version-at-all', '1.0.0')).toBeNull()
+    expect(compareVersion('cargo', 'not-a-version-at-all', '1.0.0')).toBeNull()
   })
 
-  it('returns null for short / lossy npm versions instead of coercing', () => {
+  it('returns null for short / lossy versions instead of coercing', () => {
     // Under-flag over mis-flag: semver.coerce would map "1.2" → "1.2.0" and
     // "1.2-junk-3" → "1.2.3", which can flip has_critical_vulnerability on
     // a malformed introduced/fixed boundary. We prefer null (no match).
@@ -76,7 +86,6 @@ describe('compareVersion — maven (ComparableVersion-style)', () => {
 describe('compareVersion — unsupported ecosystems', () => {
   it('returns null for ecosystems we have no comparator for', () => {
     expect(compareVersion('PyPI', '1.0.0', '2.0.0')).toBeNull()
-    expect(compareVersion('crates.io', '0.1', '0.2')).toBeNull()
   })
 
   it('rejects titlecase "Maven" — production storage is always lowercase', () => {

services/apps/packages_worker/src/osv/fetchEcosystemZip.ts

@@ -70,7 +70,8 @@ export async function* fetchEcosystemZip(
   await mkdir(ecoDir, { recursive: true })
   const zipPath = path.join(ecoDir, 'all.zip')
 
-  const url = `${baseUrl.replace(/\/$/, '')}/${ecosystem}/all.zip`
+  const bucketEcosystem = ecosystem === 'cargo' ? 'crates.io' : ecosystem
+  const url = `${baseUrl.replace(/\/$/, '')}/${bucketEcosystem}/all.zip`
 
   try {
     await downloadZip(url, zipPath)

services/apps/packages_worker/src/osv/parseOsvRecord.ts

@@ -111,7 +111,8 @@ export function parseOsvRecord(
   for (const affected of record.affected ?? []) {
     const pkg = affected.package
     if (!pkg) continue
-    const ecosystem = pkg.ecosystem.toLowerCase()
+    const rawEcosystem = pkg.ecosystem.toLowerCase()
+    const ecosystem = rawEcosystem === 'crates.io' ? 'cargo' : rawEcosystem
     if (!allowedEcosystems.has(ecosystem)) continue
 
     const ranges = flattenRanges(affected)

services/apps/packages_worker/src/osv/schedule.ts

@@ -11,7 +11,7 @@ const SCHEDULE_ID = 'osv-advisories-sync'
 // validate the env input against this list and refuse to register the
 // schedule on a mismatch — better a loud startup error than a silent miss.
 // Add new entries here when v1 expands beyond npm + Maven.
-const VALID_ECOSYSTEMS = ['npm', 'Maven'] as const
+const VALID_ECOSYSTEMS = ['npm', 'Maven', 'cargo'] as const
 
 function getEcosystems(): string[] {
   const raw = process.env.OSV_ECOSYSTEMS