Merge branch 'main' into ci/CM-968-api-e2e-tests

Author: skwowet

backend/src/api/public/v1/akrites-external/index.ts

@@ -12,26 +12,39 @@ import { getAkritesExternalContactDetailBatch } from '../packages/getAkritesExte
 import { getAkritesExternalPackageDetail } from '../packages/getAkritesExternalPackageDetail'
 import { getAkritesExternalPackageDetailBatch } from '../packages/getAkritesExternalPackageDetailBatch'
 import { getBlastRadiusJob } from '../packages/getBlastRadiusJob'
+import { ingestAkritesExternalContactDetail } from '../packages/ingestAkritesExternalContactDetail'
 import { submitBlastRadiusJob } from '../packages/submitBlastRadiusJob'
 
 const rateLimiter = createRateLimiter({ max: 60, windowMs: 60 * 1000 })
 
-// Blast-radius jobs kick off a Temporal workflow per request, so they get their own,
-// much stricter limiter — configurable via env so it can be tuned without a redeploy.
-// Defaults to 5 requests/hour.
-const blastRadiusRateLimitMax = Number(process.env.AKRITES_BLAST_RADIUS_RATE_LIMIT_MAX)
-const blastRadiusRateLimitWindowMs = Number(process.env.AKRITES_BLAST_RADIUS_RATE_LIMIT_WINDOW_MS)
+// Shared by every endpoint below that kicks off a Temporal workflow per request — those
+// get their own, much stricter limiter than plain reads, configurable via env so it can
+// be tuned without a redeploy.
+function envTunableRateLimiter(envPrefix: string, defaultMax: number, defaultWindowMs: number) {
+  const max = Number(process.env[`${envPrefix}_MAX`])
+  const windowMs = Number(process.env[`${envPrefix}_WINDOW_MS`])
+  return createRateLimiter({
+    max: Number.isSafeInteger(max) && max > 0 ? max : defaultMax,
+    windowMs: Number.isSafeInteger(windowMs) && windowMs > 0 ? windowMs : defaultWindowMs,
+  })
+}
+
+// Blast-radius jobs default to 5 requests/hour.
+const blastRadiusRateLimiter = envTunableRateLimiter(
+  'AKRITES_BLAST_RADIUS_RATE_LIMIT',
+  5,
+  60 * 60 * 1000,
+)
 
-const blastRadiusRateLimiter = createRateLimiter({
-  max:
-    Number.isSafeInteger(blastRadiusRateLimitMax) && blastRadiusRateLimitMax > 0
-      ? blastRadiusRateLimitMax
-      : 5,
-  windowMs:
-    Number.isSafeInteger(blastRadiusRateLimitWindowMs) && blastRadiusRateLimitWindowMs > 0
-      ? blastRadiusRateLimitWindowMs
-      : 60 * 60 * 1000,
-})
+// /contacts/ingest starts a Temporal workflow and blocks for it (worst case ~95s per
+// attempt cycle, plus unbounded time waiting for a free worker slot — see
+// security-contacts/workflows.ts's singleActs config), vs. the read-only /contacts/detail
+// endpoints, so it gets its own limiter. Defaults to 20 requests/hour.
+const contactIngestRateLimiter = envTunableRateLimiter(
+  'AKRITES_CONTACT_INGEST_RATE_LIMIT',
+  20,
+  60 * 60 * 1000,
+)
 
 export function akritesExternalRouter(): Router {
   const router = Router()
@@ -62,11 +75,33 @@ export function akritesExternalRouter(): Router {
   // them via the packages scope. That scope isn't issued by Auth0 yet, so reuse the
   // closest issued one — READ_MAINTAINER_ROLES (maintainer data) — NOT READ_PACKAGES.
   // TODO: swap for cdp:maintainers:read once issued.
+  // requireScopes is applied per-route (not router-level) so each route can put its own
+  // rate limiter *before* the scope check — failed-auth requests still count against that
+  // route's quota — without forcing every route in this subrouter onto the same limiter
+  // instance. /ingest gets its own dedicated contactIngestRateLimiter instead of sharing
+  // the read endpoints' quota, matching the blast-radius jobs endpoint below.
+  const contactsScopes = [SCOPES.READ_MAINTAINER_ROLES]
   const contactsSubRouter = Router()
-  contactsSubRouter.use(rateLimiter)
-  contactsSubRouter.use(requireScopes([SCOPES.READ_MAINTAINER_ROLES]))
-  contactsSubRouter.get('/detail', safeWrap(getAkritesExternalContactDetail))
-  contactsSubRouter.post(/^\/detail:batch\/?$/, safeWrap(getAkritesExternalContactDetailBatch))
+  contactsSubRouter.get(
+    '/detail',
+    rateLimiter,
+    requireScopes(contactsScopes),
+    safeWrap(getAkritesExternalContactDetail),
+  )
+  contactsSubRouter.post(
+    /^\/detail:batch\/?$/,
+    rateLimiter,
+    requireScopes(contactsScopes),
+    safeWrap(getAkritesExternalContactDetailBatch),
+  )
+  // Sync, single-purl on-demand ingest — starts a Temporal workflow and blocks a while,
+  // so it gets the dedicated contactIngestRateLimiter, not the shared rateLimiter above.
+  contactsSubRouter.post(
+    '/ingest',
+    contactIngestRateLimiter,
+    requireScopes(contactsScopes),
+    safeWrap(ingestAkritesExternalContactDetail),
+  )
   router.use('/contacts', contactsSubRouter)
 
   // TODO: the contract gates blast-radius behind a dedicated read:advisories scope

backend/src/api/public/v1/akrites-external/openapi.yaml

@@ -686,6 +686,9 @@ components:
         - vulnerabilityReportingUrl
         - bugBountyUrl
         - pvrEnabled
+        - declaredRepositoryUrl
+        - resolvedRepositoryUrl
+        - repoMappingConfidence
         - targetOrganizationName
         - bugBountyProgramFlag
         - reportingMethods
@@ -716,6 +719,28 @@ components:
         pvrEnabled:
           type: boolean
           nullable: true
+        declaredRepositoryUrl:
+          type: string
+          nullable: true
+          description: Raw repository URL as declared in the package's own metadata.
+        resolvedRepositoryUrl:
+          type: string
+          nullable: true
+          description: >
+            Best available repository URL, resolved via the confidence-ranked
+            package_repos → repos join — the repo CDP has actually resolved and
+            enriched (scorecard, branch protection, etc.). Null when the
+            package has no package_repos link yet.
+        repoMappingConfidence:
+          type: number
+          format: float
+          minimum: 0
+          maximum: 1
+          nullable: true
+          description: >
+            Confidence score of the package_repos mapping used to resolve
+            resolvedRepositoryUrl. Null when the package has no package_repos
+            link yet.
         targetOrganizationName:
           type: string
           nullable: true
@@ -1221,3 +1246,94 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/Error'
+
+  /akrites-external/contacts/ingest:
+    post:
+      operationId: ingestContactDetail
+      summary: On-demand security-contacts ingest for a single PURL
+      description: >
+        Synchronous, single-purl only — no batch variant, since fanning this out over
+        many purls would multiply concurrent Temporal workflow starts.
+
+        If the linked repo's security contacts have already been ingested at least
+        once, returns the existing ContactDetail immediately without triggering the
+        workflow. If the purl is unknown, or the package has no linked repo at all,
+        returns 404 immediately without triggering the workflow either. Otherwise
+        blocks while a single-repo Temporal workflow (ingestSecurityContactsForPurlWorkflow)
+        ingests the linked repo's security contacts — worst case ~95s per attempt
+        cycle, plus unbounded time waiting for a free worker slot — then returns the
+        same ContactDetail payload as GET /contacts/detail. Concurrent requests for
+        the same purl attach to the same running workflow instead of starting
+        duplicates.
+
+
+        Note: the underlying workflow itself has no "already ingested" gate — it
+        reprocesses and can replace a repo's contacts on every invocation. This
+        endpoint is what prevents repeat calls from re-triggering it for a purl
+        that's already been ingested.
+
+
+        Rate limited independently of the other /contacts endpoints — default
+        20 requests/hour, tunable via AKRITES_CONTACT_INGEST_RATE_LIMIT_MAX /
+        AKRITES_CONTACT_INGEST_RATE_LIMIT_WINDOW_MS.
+
+
+        Not yet implemented: this is a synchronous, blocking call. Future work should
+        make this async — an accept-and-poll job, similar to POST /blast-radius/jobs —
+        so callers aren't held open for the duration of the ingest.
+      tags: [Contacts]
+      security:
+        - M2MBearer:
+            - read:maintainer-roles
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              required: [purl]
+              properties:
+                purl:
+                  type: string
+                  example: pkg:npm/%40angular/core
+      responses:
+        '200':
+          description: >-
+            Security contact detail — either just ingested by this call, or
+            returned from a prior ingest without re-triggering the workflow.
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ContactDetail'
+        '400':
+          description: Malformed purl.
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+        '401':
+          description: Missing or invalid bearer token.
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+        '403':
+          description: Token missing read:maintainer-roles scope.
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+        '404':
+          description: >-
+            Purl is unknown, or the package has no linked repo to ingest
+            contacts from — returned without triggering the workflow.
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+        '429':
+          description: Too many requests — rate-limited independently of the other endpoints.
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'

backend/src/api/public/v1/packages/akritesExternalContactDetail.test.ts

@@ -15,6 +15,10 @@ function baseRow(
     vulnerabilityReportingUrl: null,
     bugBountyUrl: null,
     pvrEnabled: true,
+    declaredRepositoryUrl: 'https://github.com/lodash/lodash.git',
+    resolvedRepositoryUrl: 'https://github.com/lodash/lodash',
+    repoMappingConfidence: 0.9,
+    contactsLastRefreshed: '2024-01-01T00:00:00.000Z',
     securityContacts: [
       {
         channel: 'email',
@@ -107,4 +111,31 @@ describe('toAkritesExternalContactDetail', () => {
     expect(result.vulnerabilityReportingUrl).toBe('https://example.org/report')
     expect(result.pvrEnabled).toBe(false)
   })
+
+  it('passes through the repo provenance fields when present', () => {
+    const result = toAkritesExternalContactDetail(baseRow())
+    expect(result.declaredRepositoryUrl).toBe('https://github.com/lodash/lodash.git')
+    expect(result.resolvedRepositoryUrl).toBe('https://github.com/lodash/lodash')
+    expect(result.repoMappingConfidence).toBe(0.9)
+  })
+
+  it('casts repoMappingConfidence from a numeric string (pg-promise numeric type)', () => {
+    const result = toAkritesExternalContactDetail(
+      baseRow({ repoMappingConfidence: '0.9' as unknown as number }),
+    )
+    expect(result.repoMappingConfidence).toBe(0.9)
+  })
+
+  it('returns resolvedRepositoryUrl and repoMappingConfidence as null when there is no repo link', () => {
+    const result = toAkritesExternalContactDetail(
+      baseRow({ resolvedRepositoryUrl: null, repoMappingConfidence: null }),
+    )
+    expect(result.resolvedRepositoryUrl).toBeNull()
+    expect(result.repoMappingConfidence).toBeNull()
+  })
+
+  it('returns declaredRepositoryUrl as null when the package has no declared repository', () => {
+    const result = toAkritesExternalContactDetail(baseRow({ declaredRepositoryUrl: null }))
+    expect(result.declaredRepositoryUrl).toBeNull()
+  })
 })

backend/src/api/public/v1/packages/akritesExternalContactDetail.ts

@@ -4,6 +4,8 @@ import {
   securityContactConfidenceBand,
 } from '@crowd/data-access-layer'
 
+import { toNullableNumber } from './mappers'
+
 // Both the per-contact band and the aggregate band mirror the internal
 // SecurityContactConfidence enum verbatim (PRIMARY/SECONDARY/FALLBACK/NONE) —
 // no external crosswalk (confirmed with product).
@@ -28,6 +30,9 @@ export interface AkritesExternalContactDetail {
   vulnerabilityReportingUrl: string | null
   bugBountyUrl: string | null
   pvrEnabled: boolean | null
+  declaredRepositoryUrl: string | null
+  resolvedRepositoryUrl: string | null
+  repoMappingConfidence: number | null
   // Reserved by the contract but not stored today — always null (see the akrites-external
   // OpenAPI notes). Typed as null so a future non-null shape is an explicit, reviewed change.
   targetOrganizationName: null
@@ -71,6 +76,9 @@ export function toAkritesExternalContactDetail(
     vulnerabilityReportingUrl: row.vulnerabilityReportingUrl ?? null,
     bugBountyUrl: row.bugBountyUrl ?? null,
     pvrEnabled: row.pvrEnabled ?? null,
+    declaredRepositoryUrl: row.declaredRepositoryUrl ?? null,
+    resolvedRepositoryUrl: row.resolvedRepositoryUrl ?? null,
+    repoMappingConfidence: toNullableNumber(row.repoMappingConfidence),
     targetOrganizationName: null,
     bugBountyProgramFlag: null,
     reportingMethods: null,

backend/src/api/public/v1/packages/akritesExternalPackageDetail.ts

@@ -4,7 +4,7 @@ import {
   computeHealthBand,
 } from '@crowd/data-access-layer'
 
-import { repoMappingLabel, snakeToCamelKeys } from './mappers'
+import { repoMappingLabel, snakeToCamelKeys, toNullableNumber } from './mappers'
 import { HEALTH_BAND_SET, LIFECYCLE_VALUES, type Lifecycle } from './types'
 
 const LIFECYCLE_SET = new Set<string>(LIFECYCLE_VALUES)
@@ -102,8 +102,7 @@ export function toAkritesExternalPackageDetail(
   row: AkritesExternalPackageDetailRow,
 ): AkritesExternalPackageDetail {
   const scorecardScore = row.scorecardScore != null ? Number(row.scorecardScore) : null
-  const mappingConfidence =
-    row.repoMappingConfidence != null ? Number(row.repoMappingConfidence) : null
+  const mappingConfidence = toNullableNumber(row.repoMappingConfidence)
 
   return {
     purl: row.purl,

backend/src/api/public/v1/packages/getPackage.ts

@@ -13,7 +13,7 @@ import { getPackagesQx } from '@/db/packagesDb'
 import { ok } from '@/utils/api'
 import { validateOrThrow } from '@/utils/validation'
 
-import { repoMappingLabel, snakeToCamelKeys } from './mappers'
+import { repoMappingLabel, snakeToCamelKeys, toNullableNumber } from './mappers'
 import { purlQuerySchema } from './purl'
 import { HEALTH_BAND_SET, LIFECYCLE_VALUES, type StewardshipStatus } from './types'
 
@@ -35,8 +35,7 @@ export async function getPackage(req: Request, res: Response): Promise<void> {
   ])
 
   const scorecardScore = pkg.scorecardScore != null ? Number(pkg.scorecardScore) : null
-  const mappingConfidence =
-    pkg.repoMappingConfidence != null ? Number(pkg.repoMappingConfidence) : null
+  const mappingConfidence = toNullableNumber(pkg.repoMappingConfidence)
 
   const securityContacts =
     pkg.contactsLastRefreshed == null