Skip to content

Commit 8a7f246

Browse files
authored
feat: add refresh security contats api (CM-1348) (#4393)
Signed-off-by: Umberto Sgueglia <usgueglia@contractor.linuxfoundation.org>
1 parent 04214f7 commit 8a7f246

8 files changed

Lines changed: 418 additions & 20 deletions

File tree

backend/src/api/public/v1/akrites-external/index.ts

Lines changed: 54 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -12,26 +12,39 @@ import { getAkritesExternalContactDetailBatch } from '../packages/getAkritesExte
1212
import { getAkritesExternalPackageDetail } from '../packages/getAkritesExternalPackageDetail'
1313
import { getAkritesExternalPackageDetailBatch } from '../packages/getAkritesExternalPackageDetailBatch'
1414
import { getBlastRadiusJob } from '../packages/getBlastRadiusJob'
15+
import { ingestAkritesExternalContactDetail } from '../packages/ingestAkritesExternalContactDetail'
1516
import { submitBlastRadiusJob } from '../packages/submitBlastRadiusJob'
1617

1718
const rateLimiter = createRateLimiter({ max: 60, windowMs: 60 * 1000 })
1819

19-
// Blast-radius jobs kick off a Temporal workflow per request, so they get their own,
20-
// much stricter limiter — configurable via env so it can be tuned without a redeploy.
21-
// Defaults to 5 requests/hour.
22-
const blastRadiusRateLimitMax = Number(process.env.AKRITES_BLAST_RADIUS_RATE_LIMIT_MAX)
23-
const blastRadiusRateLimitWindowMs = Number(process.env.AKRITES_BLAST_RADIUS_RATE_LIMIT_WINDOW_MS)
20+
// Shared by every endpoint below that kicks off a Temporal workflow per request — those
21+
// get their own, much stricter limiter than plain reads, configurable via env so it can
22+
// be tuned without a redeploy.
23+
function envTunableRateLimiter(envPrefix: string, defaultMax: number, defaultWindowMs: number) {
24+
const max = Number(process.env[`${envPrefix}_MAX`])
25+
const windowMs = Number(process.env[`${envPrefix}_WINDOW_MS`])
26+
return createRateLimiter({
27+
max: Number.isSafeInteger(max) && max > 0 ? max : defaultMax,
28+
windowMs: Number.isSafeInteger(windowMs) && windowMs > 0 ? windowMs : defaultWindowMs,
29+
})
30+
}
31+
32+
// Blast-radius jobs default to 5 requests/hour.
33+
const blastRadiusRateLimiter = envTunableRateLimiter(
34+
'AKRITES_BLAST_RADIUS_RATE_LIMIT',
35+
5,
36+
60 * 60 * 1000,
37+
)
2438

25-
const blastRadiusRateLimiter = createRateLimiter({
26-
max:
27-
Number.isSafeInteger(blastRadiusRateLimitMax) && blastRadiusRateLimitMax > 0
28-
? blastRadiusRateLimitMax
29-
: 5,
30-
windowMs:
31-
Number.isSafeInteger(blastRadiusRateLimitWindowMs) && blastRadiusRateLimitWindowMs > 0
32-
? blastRadiusRateLimitWindowMs
33-
: 60 * 60 * 1000,
34-
})
39+
// /contacts/ingest starts a Temporal workflow and blocks for it (worst case ~95s per
40+
// attempt cycle, plus unbounded time waiting for a free worker slot — see
41+
// security-contacts/workflows.ts's singleActs config), vs. the read-only /contacts/detail
42+
// endpoints, so it gets its own limiter. Defaults to 20 requests/hour.
43+
const contactIngestRateLimiter = envTunableRateLimiter(
44+
'AKRITES_CONTACT_INGEST_RATE_LIMIT',
45+
20,
46+
60 * 60 * 1000,
47+
)
3548

3649
export function akritesExternalRouter(): Router {
3750
const router = Router()
@@ -62,11 +75,33 @@ export function akritesExternalRouter(): Router {
6275
// them via the packages scope. That scope isn't issued by Auth0 yet, so reuse the
6376
// closest issued one — READ_MAINTAINER_ROLES (maintainer data) — NOT READ_PACKAGES.
6477
// TODO: swap for cdp:maintainers:read once issued.
78+
// requireScopes is applied per-route (not router-level) so each route can put its own
79+
// rate limiter *before* the scope check — failed-auth requests still count against that
80+
// route's quota — without forcing every route in this subrouter onto the same limiter
81+
// instance. /ingest gets its own dedicated contactIngestRateLimiter instead of sharing
82+
// the read endpoints' quota, matching the blast-radius jobs endpoint below.
83+
const contactsScopes = [SCOPES.READ_MAINTAINER_ROLES]
6584
const contactsSubRouter = Router()
66-
contactsSubRouter.use(rateLimiter)
67-
contactsSubRouter.use(requireScopes([SCOPES.READ_MAINTAINER_ROLES]))
68-
contactsSubRouter.get('/detail', safeWrap(getAkritesExternalContactDetail))
69-
contactsSubRouter.post(/^\/detail:batch\/?$/, safeWrap(getAkritesExternalContactDetailBatch))
85+
contactsSubRouter.get(
86+
'/detail',
87+
rateLimiter,
88+
requireScopes(contactsScopes),
89+
safeWrap(getAkritesExternalContactDetail),
90+
)
91+
contactsSubRouter.post(
92+
/^\/detail:batch\/?$/,
93+
rateLimiter,
94+
requireScopes(contactsScopes),
95+
safeWrap(getAkritesExternalContactDetailBatch),
96+
)
97+
// Sync, single-purl on-demand ingest — starts a Temporal workflow and blocks a while,
98+
// so it gets the dedicated contactIngestRateLimiter, not the shared rateLimiter above.
99+
contactsSubRouter.post(
100+
'/ingest',
101+
contactIngestRateLimiter,
102+
requireScopes(contactsScopes),
103+
safeWrap(ingestAkritesExternalContactDetail),
104+
)
70105
router.use('/contacts', contactsSubRouter)
71106

72107
// TODO: the contract gates blast-radius behind a dedicated read:advisories scope

backend/src/api/public/v1/akrites-external/openapi.yaml

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1246,3 +1246,94 @@ paths:
12461246
application/json:
12471247
schema:
12481248
$ref: '#/components/schemas/Error'
1249+
1250+
/akrites-external/contacts/ingest:
1251+
post:
1252+
operationId: ingestContactDetail
1253+
summary: On-demand security-contacts ingest for a single PURL
1254+
description: >
1255+
Synchronous, single-purl only — no batch variant, since fanning this out over
1256+
many purls would multiply concurrent Temporal workflow starts.
1257+
1258+
If the linked repo's security contacts have already been ingested at least
1259+
once, returns the existing ContactDetail immediately without triggering the
1260+
workflow. If the purl is unknown, or the package has no linked repo at all,
1261+
returns 404 immediately without triggering the workflow either. Otherwise
1262+
blocks while a single-repo Temporal workflow (ingestSecurityContactsForPurlWorkflow)
1263+
ingests the linked repo's security contacts — worst case ~95s per attempt
1264+
cycle, plus unbounded time waiting for a free worker slot — then returns the
1265+
same ContactDetail payload as GET /contacts/detail. Concurrent requests for
1266+
the same purl attach to the same running workflow instead of starting
1267+
duplicates.
1268+
1269+
1270+
Note: the underlying workflow itself has no "already ingested" gate — it
1271+
reprocesses and can replace a repo's contacts on every invocation. This
1272+
endpoint is what prevents repeat calls from re-triggering it for a purl
1273+
that's already been ingested.
1274+
1275+
1276+
Rate limited independently of the other /contacts endpoints — default
1277+
20 requests/hour, tunable via AKRITES_CONTACT_INGEST_RATE_LIMIT_MAX /
1278+
AKRITES_CONTACT_INGEST_RATE_LIMIT_WINDOW_MS.
1279+
1280+
1281+
Not yet implemented: this is a synchronous, blocking call. Future work should
1282+
make this async — an accept-and-poll job, similar to POST /blast-radius/jobs —
1283+
so callers aren't held open for the duration of the ingest.
1284+
tags: [Contacts]
1285+
security:
1286+
- M2MBearer:
1287+
- read:maintainer-roles
1288+
requestBody:
1289+
required: true
1290+
content:
1291+
application/json:
1292+
schema:
1293+
type: object
1294+
required: [purl]
1295+
properties:
1296+
purl:
1297+
type: string
1298+
example: pkg:npm/%40angular/core
1299+
responses:
1300+
'200':
1301+
description: >-
1302+
Security contact detail — either just ingested by this call, or
1303+
returned from a prior ingest without re-triggering the workflow.
1304+
content:
1305+
application/json:
1306+
schema:
1307+
$ref: '#/components/schemas/ContactDetail'
1308+
'400':
1309+
description: Malformed purl.
1310+
content:
1311+
application/json:
1312+
schema:
1313+
$ref: '#/components/schemas/Error'
1314+
'401':
1315+
description: Missing or invalid bearer token.
1316+
content:
1317+
application/json:
1318+
schema:
1319+
$ref: '#/components/schemas/Error'
1320+
'403':
1321+
description: Token missing read:maintainer-roles scope.
1322+
content:
1323+
application/json:
1324+
schema:
1325+
$ref: '#/components/schemas/Error'
1326+
'404':
1327+
description: >-
1328+
Purl is unknown, or the package has no linked repo to ingest
1329+
contacts from — returned without triggering the workflow.
1330+
content:
1331+
application/json:
1332+
schema:
1333+
$ref: '#/components/schemas/Error'
1334+
'429':
1335+
description: Too many requests — rate-limited independently of the other endpoints.
1336+
content:
1337+
application/json:
1338+
schema:
1339+
$ref: '#/components/schemas/Error'

backend/src/api/public/v1/packages/akritesExternalContactDetail.test.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ function baseRow(
1818
declaredRepositoryUrl: 'https://github.com/lodash/lodash.git',
1919
resolvedRepositoryUrl: 'https://github.com/lodash/lodash',
2020
repoMappingConfidence: 0.9,
21+
contactsLastRefreshed: '2024-01-01T00:00:00.000Z',
2122
securityContacts: [
2223
{
2324
channel: 'email',
Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
import type { Request, Response } from 'express'
2+
import { describe, expect, it, vi } from 'vitest'
3+
4+
import type { AkritesExternalContactDetailRow } from '@crowd/data-access-layer'
5+
import { WorkflowIdConflictPolicy, WorkflowIdReusePolicy } from '@crowd/temporal'
6+
7+
import { ingestAkritesExternalContactDetail } from './ingestAkritesExternalContactDetail'
8+
9+
const { execute, getContactDetailsByPurls } = vi.hoisted(() => ({
10+
execute: vi.fn(),
11+
getContactDetailsByPurls: vi.fn(),
12+
}))
13+
14+
vi.mock('@/db/packagesTemporal', () => ({
15+
getPackagesTemporalClient: vi.fn().mockResolvedValue({ workflow: { execute } }),
16+
}))
17+
18+
vi.mock('@/db/packagesDb', () => ({
19+
getPackagesQx: vi.fn().mockResolvedValue({}),
20+
}))
21+
22+
vi.mock('@crowd/data-access-layer', () => ({
23+
getContactDetailsByPurls,
24+
}))
25+
26+
function baseRow(
27+
overrides: Partial<AkritesExternalContactDetailRow> = {},
28+
): AkritesExternalContactDetailRow {
29+
return {
30+
purl: 'pkg:npm/lodash',
31+
name: 'lodash',
32+
ecosystem: 'npm',
33+
securityPolicyUrl: null,
34+
vulnerabilityReportingUrl: null,
35+
bugBountyUrl: null,
36+
pvrEnabled: null,
37+
declaredRepositoryUrl: null,
38+
resolvedRepositoryUrl: null,
39+
repoMappingConfidence: null,
40+
contactsLastRefreshed: null,
41+
securityContacts: [],
42+
...overrides,
43+
}
44+
}
45+
46+
function mockReqRes(body: unknown) {
47+
execute.mockClear()
48+
getContactDetailsByPurls.mockClear()
49+
50+
const req = { body } as unknown as Request
51+
52+
const json = vi.fn()
53+
const status = vi.fn().mockReturnValue({ json })
54+
const res = { status, json } as unknown as Response
55+
56+
return { req, res, status, json }
57+
}
58+
59+
describe('ingestAkritesExternalContactDetail', () => {
60+
it('returns the existing contact detail without triggering the workflow when already ingested', async () => {
61+
getContactDetailsByPurls.mockResolvedValue([
62+
baseRow({ contactsLastRefreshed: '2024-01-01T00:00:00.000Z' }),
63+
])
64+
65+
const { req, res, json } = mockReqRes({ purl: 'pkg:npm/lodash' })
66+
67+
await ingestAkritesExternalContactDetail(req, res)
68+
69+
expect(execute).not.toHaveBeenCalled()
70+
expect(getContactDetailsByPurls).toHaveBeenCalledTimes(1)
71+
expect(json).toHaveBeenCalledWith(expect.objectContaining({ purl: 'pkg:npm/lodash' }))
72+
})
73+
74+
it('executes ingestSecurityContactsForPurlWorkflow and returns the re-read contact detail when never ingested', async () => {
75+
execute.mockResolvedValue({ found: true, repoId: 'repo-1' })
76+
getContactDetailsByPurls
77+
.mockResolvedValueOnce([
78+
baseRow({
79+
resolvedRepositoryUrl: 'https://github.com/lodash/lodash',
80+
contactsLastRefreshed: null,
81+
}),
82+
])
83+
.mockResolvedValueOnce([baseRow({ contactsLastRefreshed: '2024-01-01T00:00:00.000Z' })])
84+
85+
const { req, res, json } = mockReqRes({ purl: 'pkg:npm/lodash' })
86+
87+
await ingestAkritesExternalContactDetail(req, res)
88+
89+
expect(execute).toHaveBeenCalledTimes(1)
90+
const [workflowType, options] = execute.mock.calls[0]
91+
expect(workflowType).toBe('ingestSecurityContactsForPurlWorkflow')
92+
expect(options.taskQueue).toBe('security-contacts-worker')
93+
expect(options.workflowId).toMatch(/^security-contacts-ondemand:[0-9a-f]{64}$/)
94+
expect(options.workflowIdConflictPolicy).toBe(WorkflowIdConflictPolicy.USE_EXISTING)
95+
expect(options.workflowIdReusePolicy).toBe(WorkflowIdReusePolicy.ALLOW_DUPLICATE)
96+
expect(options.args).toEqual(['pkg:npm/lodash'])
97+
98+
expect(getContactDetailsByPurls).toHaveBeenCalledTimes(2)
99+
expect(getContactDetailsByPurls).toHaveBeenCalledWith(expect.anything(), ['pkg:npm/lodash'])
100+
expect(json).toHaveBeenCalledWith(expect.objectContaining({ purl: 'pkg:npm/lodash' }))
101+
})
102+
103+
it('derives the same deterministic workflowId for the same purl', async () => {
104+
execute.mockResolvedValue({ found: true })
105+
getContactDetailsByPurls.mockResolvedValue([
106+
baseRow({
107+
resolvedRepositoryUrl: 'https://github.com/lodash/lodash',
108+
contactsLastRefreshed: null,
109+
}),
110+
])
111+
112+
const { req: req1, res: res1 } = mockReqRes({ purl: 'pkg:npm/lodash' })
113+
await ingestAkritesExternalContactDetail(req1, res1)
114+
const id1 = execute.mock.calls[0][1].workflowId
115+
116+
const { req: req2, res: res2 } = mockReqRes({ purl: 'pkg:npm/lodash' })
117+
await ingestAkritesExternalContactDetail(req2, res2)
118+
const id2 = execute.mock.calls[0][1].workflowId
119+
120+
expect(id1).toBe(id2)
121+
})
122+
123+
it('throws NotFoundError without executing the workflow when the purl is unknown', async () => {
124+
getContactDetailsByPurls.mockResolvedValue([])
125+
126+
const { req, res } = mockReqRes({ purl: 'pkg:npm/left-pad' })
127+
128+
await expect(ingestAkritesExternalContactDetail(req, res)).rejects.toThrow()
129+
expect(execute).not.toHaveBeenCalled()
130+
expect(getContactDetailsByPurls).toHaveBeenCalledTimes(1)
131+
})
132+
133+
it('throws NotFoundError without executing the workflow when the package has no linked repo', async () => {
134+
getContactDetailsByPurls.mockResolvedValue([
135+
baseRow({ resolvedRepositoryUrl: null, contactsLastRefreshed: null }),
136+
])
137+
138+
const { req, res } = mockReqRes({ purl: 'pkg:npm/left-pad' })
139+
140+
await expect(ingestAkritesExternalContactDetail(req, res)).rejects.toThrow()
141+
expect(execute).not.toHaveBeenCalled()
142+
expect(getContactDetailsByPurls).toHaveBeenCalledTimes(1)
143+
})
144+
145+
it('throws NotFoundError when the workflow reports no linked repo', async () => {
146+
execute.mockResolvedValue({ found: false })
147+
getContactDetailsByPurls.mockResolvedValue([
148+
baseRow({
149+
resolvedRepositoryUrl: 'https://github.com/example/left-pad',
150+
contactsLastRefreshed: null,
151+
}),
152+
])
153+
154+
const { req, res } = mockReqRes({ purl: 'pkg:npm/left-pad' })
155+
156+
await expect(ingestAkritesExternalContactDetail(req, res)).rejects.toThrow()
157+
expect(getContactDetailsByPurls).toHaveBeenCalledTimes(1)
158+
})
159+
160+
it('throws NotFoundError when the re-read finds no row', async () => {
161+
execute.mockResolvedValue({ found: true })
162+
getContactDetailsByPurls
163+
.mockResolvedValueOnce([
164+
baseRow({
165+
resolvedRepositoryUrl: 'https://github.com/lodash/lodash',
166+
contactsLastRefreshed: null,
167+
}),
168+
])
169+
.mockResolvedValueOnce([])
170+
171+
const { req, res } = mockReqRes({ purl: 'pkg:npm/lodash' })
172+
173+
await expect(ingestAkritesExternalContactDetail(req, res)).rejects.toThrow()
174+
})
175+
176+
it('rejects a request missing purl without executing a workflow', async () => {
177+
const { req, res } = mockReqRes({})
178+
179+
await expect(ingestAkritesExternalContactDetail(req, res)).rejects.toThrow()
180+
expect(execute).not.toHaveBeenCalled()
181+
})
182+
})

0 commit comments

Comments
 (0)