Skip to content

Commit ac4dc81

Browse files
authored
Merge branch 'main' into fix/duplicated_repos
2 parents eff5795 + e021298 commit ac4dc81

10 files changed

Lines changed: 193 additions & 178 deletions

File tree

backend/src/database/repositories/memberRepository.ts

Lines changed: 4 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import lodash, { chunk, uniq } from 'lodash'
1+
import lodash, { uniq } from 'lodash'
22
import Sequelize, { QueryTypes } from 'sequelize'
33

44
import {
@@ -29,7 +29,7 @@ import {
2929
} from '@crowd/data-access-layer'
3030
import { findManyLfxMemberships } from '@crowd/data-access-layer/src/lfx_memberships'
3131
import { findMaintainerRoles } from '@crowd/data-access-layer/src/maintainers'
32-
import { addMemberNoMerge, removeMemberToMerge } from '@crowd/data-access-layer/src/member_merge'
32+
import { insertMemberNoMerge, removeMemberToMerge } from '@crowd/data-access-layer/src/member_merge'
3333
import {
3434
deleteMemberSegmentAffiliations,
3535
findMemberAffiliations,
@@ -86,7 +86,7 @@ import MemberAttributeSettingsRepository from './memberAttributeSettingsReposito
8686
import SegmentRepository from './segmentRepository'
8787
import SequelizeRepository from './sequelizeRepository'
8888
import TenantRepository from './tenantRepository'
89-
import { IMemberMergeSuggestion, mapUsernameToIdentities } from './types/memberTypes'
89+
import { mapUsernameToIdentities } from './types/memberTypes'
9090

9191
const { Op } = Sequelize
9292

@@ -581,72 +581,6 @@ class MemberRepository {
581581
}
582582
}
583583

584-
static async addToMerge(
585-
suggestions: IMemberMergeSuggestion[],
586-
options: IRepositoryOptions,
587-
): Promise<void> {
588-
const transaction = SequelizeRepository.getTransaction(options)
589-
const seq = SequelizeRepository.getSequelize(options)
590-
591-
// Remove possible duplicates
592-
suggestions = lodash.uniqWith(suggestions, (a, b) =>
593-
lodash.isEqual(lodash.sortBy(a.members), lodash.sortBy(b.members)),
594-
)
595-
596-
// Process suggestions in chunks of 100 or less
597-
const suggestionChunks = chunk(suggestions, 100)
598-
599-
const insertValues = (
600-
memberId: string,
601-
toMergeId: string,
602-
similarity: number | null,
603-
index: number,
604-
) => {
605-
const idPlaceholder = (key: string) => `${key}${index}`
606-
return {
607-
query: `(:${idPlaceholder('memberId')}, :${idPlaceholder('toMergeId')}, :${idPlaceholder(
608-
'similarity',
609-
)}, NOW(), NOW())`,
610-
replacements: {
611-
[idPlaceholder('memberId')]: memberId,
612-
[idPlaceholder('toMergeId')]: toMergeId,
613-
[idPlaceholder('similarity')]: similarity === null ? null : similarity,
614-
},
615-
}
616-
}
617-
618-
for (const suggestionChunk of suggestionChunks) {
619-
const placeholders: string[] = []
620-
let replacements: Record<string, unknown> = {}
621-
622-
suggestionChunk.forEach((suggestion, index) => {
623-
const { query, replacements: chunkReplacements } = insertValues(
624-
suggestion.members[0],
625-
suggestion.members[1],
626-
suggestion.similarity,
627-
index,
628-
)
629-
placeholders.push(query)
630-
replacements = { ...replacements, ...chunkReplacements }
631-
})
632-
633-
const query = `
634-
INSERT INTO "memberToMerge" ("memberId", "toMergeId", "similarity", "createdAt", "updatedAt")
635-
VALUES ${placeholders.join(', ')} on conflict do nothing;
636-
`
637-
try {
638-
await seq.query(query, {
639-
replacements,
640-
type: QueryTypes.INSERT,
641-
transaction,
642-
})
643-
} catch (error) {
644-
options.log.error('error adding members to merge', error)
645-
throw error
646-
}
647-
}
648-
}
649-
650584
static async removeToMerge(id, toMergeId, options: IRepositoryOptions) {
651585
const qx = SequelizeRepository.getQueryExecutor(options)
652586

@@ -656,7 +590,7 @@ class MemberRepository {
656590
static async addNoMerge(id, toMergeId, options: IRepositoryOptions) {
657591
const qx = SequelizeRepository.getQueryExecutor(options)
658592

659-
await addMemberNoMerge(qx, id, toMergeId)
593+
await insertMemberNoMerge(qx, id, toMergeId)
660594
}
661595

662596
static async memberExists(
Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
-- Blast-radius analysis pipeline (CM-1328). Mirrors the 4-stage PoC
2+
-- (linux-foundation/blast-radius): intel -> dependents -> reachability -> report.
3+
-- Everything the PoC kept as JSON files under data/<VULN-ID>/ moves to these
4+
-- tables so runs are resumable, queryable, and inspectable across ecosystems.
5+
-- The aggregated report (naive vs true blast radius, bucket counts) is not
6+
-- materialized here — it's derived from blast_radius_verdicts at read time.
7+
8+
-- One row per submitted job (backend's submitBlastRadiusJob). id matches the
9+
-- analysisId the API already generates before starting the Temporal workflow.
10+
CREATE TABLE IF NOT EXISTS blast_radius_analyses (
11+
id UUID PRIMARY KEY,
12+
-- Raw GHSA/CVE id from the request; the advisory may not be synced from
13+
-- deps.dev BQ yet, so advisory_id is resolved (and filled in) later.
14+
advisory_osv_id TEXT NOT NULL,
15+
advisory_id BIGINT REFERENCES advisories (id),
16+
-- Raw value from the request (purl or bare name); may not resolve to a
17+
-- known package yet. package_id is filled in once/if it resolves.
18+
package_name TEXT,
19+
package_id BIGINT REFERENCES packages (id),
20+
ecosystem TEXT NOT NULL,
21+
status TEXT NOT NULL DEFAULT 'pending'
22+
CHECK (status IN ('pending', 'running', 'done', 'failed')),
23+
force BOOLEAN NOT NULL DEFAULT FALSE,
24+
-- Stage 2 (dependents) run metadata — the per-dependent rows live in
25+
-- blast_radius_dependents; these are the population-level numbers report
26+
-- rendering needs alongside them (dependents_doc.source / candidates_considered).
27+
dependents_source TEXT,
28+
candidates_considered INT,
29+
total_cost_usd NUMERIC(10, 4) NOT NULL DEFAULT 0,
30+
error TEXT,
31+
started_at TIMESTAMPTZ,
32+
completed_at TIMESTAMPTZ,
33+
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
34+
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
35+
);
36+
37+
CREATE INDEX IF NOT EXISTS blast_radius_analyses_advisory_id_idx
38+
ON blast_radius_analyses (advisory_id);
39+
CREATE INDEX IF NOT EXISTS blast_radius_analyses_package_id_idx
40+
ON blast_radius_analyses (package_id) WHERE package_id IS NOT NULL;
41+
CREATE INDEX IF NOT EXISTS blast_radius_analyses_status_idx
42+
ON blast_radius_analyses (status);
43+
44+
-- Stage 1 output — one row per analysis (symbol_spec.json equivalent).
45+
CREATE TABLE IF NOT EXISTS blast_radius_symbol_specs (
46+
id BIGSERIAL PRIMARY KEY,
47+
analysis_id UUID NOT NULL UNIQUE REFERENCES blast_radius_analyses (id),
48+
vuln_id TEXT NOT NULL,
49+
aliases TEXT[],
50+
package TEXT NOT NULL,
51+
ecosystem TEXT NOT NULL,
52+
affected_ranges JSONB NOT NULL,
53+
vulnerable_versions TEXT[] NOT NULL,
54+
analyzed_version TEXT NOT NULL,
55+
related_affected_packages TEXT[],
56+
vulnerable_symbols JSONB NOT NULL, -- [{name, kind, defined_in, exported_as[], notes}]
57+
import_signatures JSONB NOT NULL, -- {style: [signature, ...]}
58+
exploit_preconditions TEXT,
59+
reachability_notes TEXT,
60+
confidence NUMERIC(3, 2) NOT NULL CHECK (confidence BETWEEN 0.00 AND 1.00),
61+
sources TEXT[],
62+
summary TEXT,
63+
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
64+
);
65+
66+
-- Stage 2 output — one row per candidate dependent, analyzed or excluded.
67+
-- Unifies dependents.json's "analyzed" and "excluded_by_range" lists: the
68+
-- excluded ones never got a resolved version/tarball, hence those columns
69+
-- are nullable.
70+
CREATE TABLE IF NOT EXISTS blast_radius_dependents (
71+
id BIGSERIAL PRIMARY KEY,
72+
analysis_id UUID NOT NULL REFERENCES blast_radius_analyses (id),
73+
package_id BIGINT REFERENCES packages (id),
74+
name TEXT NOT NULL,
75+
version TEXT,
76+
downloads BIGINT,
77+
declared_range TEXT,
78+
dependency_kind TEXT, -- 'dependencies' | 'peerDependencies' | 'optionalDependencies'
79+
range_includes_vuln BOOLEAN,
80+
range_check TEXT, -- 'matched' | 'excluded' | 'unparseable-included'
81+
tarball_url TEXT,
82+
excluded_by_range BOOLEAN NOT NULL DEFAULT FALSE,
83+
exclusion_reason TEXT,
84+
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
85+
UNIQUE (analysis_id, name)
86+
);
87+
88+
CREATE INDEX IF NOT EXISTS blast_radius_dependents_analysis_id_idx
89+
ON blast_radius_dependents (analysis_id);
90+
CREATE INDEX IF NOT EXISTS blast_radius_dependents_package_id_idx
91+
ON blast_radius_dependents (package_id) WHERE package_id IS NOT NULL;
92+
93+
-- Stage 3 output — one reachability verdict per analyzed (non-excluded) dependent.
94+
CREATE TABLE IF NOT EXISTS blast_radius_verdicts (
95+
id BIGSERIAL PRIMARY KEY,
96+
analysis_id UUID NOT NULL REFERENCES blast_radius_analyses (id),
97+
dependent_id BIGINT NOT NULL UNIQUE REFERENCES blast_radius_dependents (id),
98+
uses_package BOOLEAN NOT NULL,
99+
imports_vulnerable_symbol BOOLEAN NOT NULL,
100+
import_style TEXT, -- 'main-member' | 'deep-import' | 'standalone-pkg' | 'reexport' | 'none'
101+
reachable_verdict TEXT NOT NULL CHECK (reachable_verdict IN ('affected', 'not_affected', 'unclear')),
102+
confidence NUMERIC(3, 2) NOT NULL CHECK (confidence BETWEEN 0.00 AND 1.00),
103+
evidence JSONB, -- [{file, line, snippet}]
104+
reasoning TEXT,
105+
model TEXT,
106+
turns_used INT,
107+
cost_usd NUMERIC(10, 4) NOT NULL DEFAULT 0,
108+
error TEXT,
109+
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
110+
);
111+
112+
CREATE INDEX IF NOT EXISTS blast_radius_verdicts_analysis_id_idx
113+
ON blast_radius_verdicts (analysis_id);
114+
CREATE INDEX IF NOT EXISTS blast_radius_verdicts_analysis_id_verdict_idx
115+
ON blast_radius_verdicts (analysis_id, reachable_verdict);
116+
117+
-- Performance monitoring — one row per (analysis, stage), independent of the
118+
-- ecosystem-specific pipeline tables above so it keeps working unchanged as
119+
-- more ecosystems (and possibly different stage sets) come online.
120+
CREATE TABLE IF NOT EXISTS blast_radius_stage_runs (
121+
id BIGSERIAL PRIMARY KEY,
122+
analysis_id UUID NOT NULL REFERENCES blast_radius_analyses (id),
123+
stage TEXT NOT NULL CHECK (stage IN ('intel', 'dependents', 'reachability', 'report')),
124+
status TEXT NOT NULL DEFAULT 'running' CHECK (status IN ('running', 'succeeded', 'failed')),
125+
model TEXT, -- agent model used by this stage, NULL for deterministic stages
126+
started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
127+
completed_at TIMESTAMPTZ,
128+
duration_ms BIGINT,
129+
cost_usd NUMERIC(10, 4) NOT NULL DEFAULT 0,
130+
error TEXT,
131+
UNIQUE (analysis_id, stage)
132+
);
133+
134+
CREATE INDEX IF NOT EXISTS blast_radius_stage_runs_analysis_id_idx
135+
ON blast_radius_stage_runs (analysis_id);

backend/src/services/memberService.ts

Lines changed: 2 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,6 @@ import { MergeActionsRepository } from '../database/repositories/mergeActionsRep
4949
import SequelizeRepository from '../database/repositories/sequelizeRepository'
5050
import {
5151
BasicMemberIdentity,
52-
IMemberMergeSuggestion,
5352
mapUsernameToIdentities,
5453
} from '../database/repositories/types/memberTypes'
5554
import telemetryTrack from '../segment/telemetryTrack'
@@ -729,32 +728,6 @@ export default class MemberService extends LoggerBase {
729728
return data
730729
}
731730

732-
/**
733-
* Given two members, add them to the toMerge fields of each other.
734-
* It will also update the tenant's toMerge list, removing any entry that contains
735-
* the pair.
736-
* @returns Success/Error message
737-
*/
738-
async addToMerge(suggestions: IMemberMergeSuggestion[]) {
739-
const transaction = await SequelizeRepository.createTransaction(this.options)
740-
try {
741-
const searchSyncService = new SearchSyncService(this.options)
742-
743-
await MemberRepository.addToMerge(suggestions, { ...this.options, transaction })
744-
await SequelizeRepository.commitTransaction(transaction)
745-
746-
for (const suggestion of suggestions) {
747-
await searchSyncService.triggerMemberSync(suggestion.members[0])
748-
await searchSyncService.triggerMemberSync(suggestion.members[1])
749-
}
750-
return { status: 200 }
751-
} catch (error) {
752-
await SequelizeRepository.rollbackTransaction(transaction)
753-
this.log.error(error, 'Error while adding members to merge')
754-
throw error
755-
}
756-
}
757-
758731
/**
759732
* Given two members, add them to the noMerge fields of each other.
760733
* @param memberOneId ID of the first member
@@ -768,8 +741,9 @@ export default class MemberService extends LoggerBase {
768741
try {
769742
await MemberRepository.addNoMerge(memberOneId, memberTwoId, txOptions)
770743
await MemberRepository.addNoMerge(memberTwoId, memberOneId, txOptions)
744+
745+
// Removes from either order of the pair
771746
await MemberRepository.removeToMerge(memberOneId, memberTwoId, txOptions)
772-
await MemberRepository.removeToMerge(memberTwoId, memberOneId, txOptions)
773747

774748
await SequelizeRepository.commitTransaction(transaction)
775749
} catch (error) {

services/apps/merge_suggestions_worker/src/activities/memberMergeSuggestions.ts

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
import uniqBy from 'lodash.uniqby'
33

44
import { parseGitHubNoreplyEmail, parseGitLabNoreplyEmail } from '@crowd/common'
5-
import { addMemberNoMerge } from '@crowd/data-access-layer/src/member_merge'
5+
import { insertMemberNoMerge, removeMemberToMerge } from '@crowd/data-access-layer/src/member_merge'
66
import { MemberField, queryMembers } from '@crowd/data-access-layer/src/members'
77
import MemberMergeSuggestionsRepository from '@crowd/data-access-layer/src/old/apps/merge_suggestions_worker/memberMergeSuggestions.repo'
88
import { pgpQx } from '@crowd/data-access-layer/src/queryExecutor'
@@ -498,15 +498,14 @@ export async function getRawMemberMergeSuggestions(
498498
return memberMergeSuggestionsRepo.getRawMemberSuggestions(similarityFilter, limit)
499499
}
500500

501-
export async function removeMemberMergeSuggestion(
502-
suggestion: string[],
503-
table: MemberMergeSuggestionTable,
504-
): Promise<void> {
505-
const memberMergeSuggestionsRepo = new MemberMergeSuggestionsRepository(
506-
svc.postgres.writer.connection(),
507-
svc.log,
508-
)
509-
await memberMergeSuggestionsRepo.removeMemberMergeSuggestion(suggestion, table)
501+
export async function removeMemberMergeSuggestion(suggestion: string[]): Promise<void> {
502+
if (suggestion.length !== 2) {
503+
svc.log.debug(`Suggestions array must have two ids!`)
504+
return
505+
}
506+
507+
const qx = pgpQx(svc.postgres.writer.connection())
508+
await removeMemberToMerge(qx, suggestion[0], suggestion[1])
510509
}
511510

512511
export async function addMemberSuggestionToNoMerge(suggestion: string[]): Promise<void> {
@@ -516,5 +515,6 @@ export async function addMemberSuggestionToNoMerge(suggestion: string[]): Promis
516515
}
517516
const qx = pgpQx(svc.postgres.writer.connection())
518517

519-
await addMemberNoMerge(qx, suggestion[0], suggestion[1])
518+
await insertMemberNoMerge(qx, suggestion[0], suggestion[1])
519+
await insertMemberNoMerge(qx, suggestion[1], suggestion[0])
520520
}

services/apps/merge_suggestions_worker/src/workflows/mergeMembersWithLLM.ts

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { continueAsNew, proxyActivities } from '@temporalio/workflow'
22

3-
import { LLMSuggestionVerdictType, MemberMergeSuggestionTable } from '@crowd/types'
3+
import { LLMSuggestionVerdictType } from '@crowd/types'
44

55
import type * as activities from '../activities'
66
import { ILLMResult, IProcessMergeMemberSuggestionsWithLLM } from '../types'
@@ -78,11 +78,7 @@ export async function mergeMembersWithLLM(
7878

7979
if (members.length !== 2) {
8080
console.log(`Failed getting members data in suggestion. Skipping suggestion: ${suggestion}`)
81-
await removeMemberMergeSuggestion(suggestion, MemberMergeSuggestionTable.MEMBER_TO_MERGE_RAW)
82-
await removeMemberMergeSuggestion(
83-
suggestion,
84-
MemberMergeSuggestionTable.MEMBER_TO_MERGE_FILTERED,
85-
)
81+
await removeMemberMergeSuggestion(suggestion)
8682
continue
8783
}
8884

@@ -115,11 +111,7 @@ export async function mergeMembersWithLLM(
115111
console.log(
116112
`LLM doesn't think these members are the same. Removing from suggestions and adding to no merge: ${suggestion[0]} and ${suggestion[1]}!`,
117113
)
118-
await removeMemberMergeSuggestion(
119-
suggestion,
120-
MemberMergeSuggestionTable.MEMBER_TO_MERGE_FILTERED,
121-
)
122-
await removeMemberMergeSuggestion(suggestion, MemberMergeSuggestionTable.MEMBER_TO_MERGE_RAW)
114+
await removeMemberMergeSuggestion(suggestion)
123115
await addMemberSuggestionToNoMerge(suggestion)
124116
}
125117
}

services/libs/common_services/src/services/common.member.service.ts

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import {
2020
sanitizeMemberOrganizationDateRange,
2121
} from '@crowd/common'
2222
import {
23+
ActorType,
2324
MEMBER_MERGE_FIELDS,
2425
MemberField,
2526
QueryExecutor,
@@ -316,7 +317,10 @@ export class CommonMemberService extends LoggerBase {
316317
}
317318
}
318319

319-
const actorId = buildAuditLogOptions(options)?.actorId
320+
const audit = buildAuditLogOptions(options)
321+
322+
const actorId = audit?.actorId
323+
const notifyUserId = audit?.actorType === ActorType.USER ? actorId : undefined
320324

321325
const mergeActions = await queryMergeActions(this.qx, {
322326
fields: ['id', 'state'],
@@ -472,7 +476,7 @@ export class CommonMemberService extends LoggerBase {
472476
retry: {
473477
maximumAttempts: 10,
474478
},
475-
args: [originalId, toMergeId, original.displayName, toMerge.displayName, actorId],
479+
args: [originalId, toMergeId, original.displayName, toMerge.displayName, notifyUserId],
476480
searchAttributes: {
477481
TenantId: [DEFAULT_TENANT_ID],
478482
},

0 commit comments

Comments
 (0)