Skip to content

Commit 5f5e903

Browse files
mbani01mbani01
committed
feat: committees implementation (WIP)
Signed-off-by: Mouad BANI <mouad-mb@outlook.com>
1 parent e885a25 commit 5f5e903

11 files changed

Lines changed: 344 additions & 0 deletions

File tree

‎backend/src/database/migrations/U1775064222__addCommitteesActivityTypes.sql‎

Whitespace-only changes.

‎backend/src/database/migrations/V1775064222__addCommitteesActivityTypes.sql‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
INSERT INTO "activityTypes" ("activityType", platform, "isCodeContribution", "isCollaboration", description, "label") VALUES
2+
('added-to-committee', 'committees', false, false, 'Member is added to a committee', 'Added to committee'),
3+
('removed-from-committee', 'committees', false, false, 'Member is removed from a committee', 'Removed from committee');

‎services/apps/snowflake_connectors/src/integrations/committees/committees/buildSourceQuery.ts‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
import { IS_PROD_ENV } from '@crowd/common'
2+
3+
// Main: FIVETRAN_INGEST.SFDC_CONNECTOR_PROD_PLATFORM.COMMUNITY__C
4+
// Joins:
5+
// - ANALYTICS.SILVER_DIM.COMMITTEE (committee metadata + project slug)
6+
// - ANALYTICS.BRONZE_KAFKA_CROWD_DEV.SEGMENTS (segment resolution)
7+
// - ANALYTICS.SILVER_DIM.USERS (member identity: email, lf_username, name)
8+
// - ANALYTICS.BRONZE_FIVETRAN_SALESFORCE_B2B.USERS (actor name + email)
9+
// - ANALYTICS.BRONZE_FIVETRAN_SALESFORCE_B2B.ACCOUNTS (org data)
10+
11+
const CDP_MATCHED_SEGMENTS = `
12+
cdp_matched_segments AS (
13+
SELECT DISTINCT
14+
s.SOURCE_ID AS sourceId,
15+
s.slug
16+
FROM ANALYTICS.BRONZE_KAFKA_CROWD_DEV.SEGMENTS s
17+
WHERE s.PARENT_SLUG IS NOT NULL
18+
AND s.GRANDPARENTS_SLUG IS NOT NULL
19+
AND s.SOURCE_ID IS NOT NULL
20+
)`
21+
22+
const ORG_ACCOUNTS = `
23+
org_accounts AS (
24+
SELECT account_id, account_name, website, domain_aliases
25+
FROM ANALYTICS.BRONZE_FIVETRAN_SALESFORCE_B2B.ACCOUNTS
26+
WHERE website IS NOT NULL
27+
)`
28+
29+
export const buildSourceQuery = (sinceTimestamp?: string): string => {
30+
let select = `
31+
SELECT
32+
c.SFID,
33+
c._FIVETRAN_DELETED AS FIVETRAN_DELETED,
34+
c.CONTACTEMAIL__C,
35+
c.CREATEDBYID,
36+
c.COLLABORATION_NAME__C,
37+
c.ACCOUNT__C,
38+
c.ROLE__C,
39+
c.CREATEDDATE::TIMESTAMP_NTZ AS CREATEDDATE,
40+
c.LASTMODIFIEDDATE::TIMESTAMP_NTZ AS LASTMODIFIEDDATE,
41+
cm.COMMITTEE_ID,
42+
cm.COMMITTEE_NAME,
43+
cm.PROJECT_ID,
44+
cm.PROJECT_NAME,
45+
cm.PROJECT_SLUG,
46+
su.EMAIL AS SU_EMAIL,
47+
su.LF_USERNAME,
48+
su.PRIMARY_SOURCE_USER_ID,
49+
su.FIRST_NAME AS SU_FIRST_NAME,
50+
su.LAST_NAME AS SU_LAST_NAME,
51+
su.FULL_NAME AS SU_FULL_NAME,
52+
bu.FIRST_NAME AS BU_FIRST_NAME,
53+
bu.LAST_NAME AS BU_LAST_NAME,
54+
bu.EMAIL AS BU_EMAIL,
55+
org.account_name AS ACCOUNT_NAME,
56+
org.website AS ORG_WEBSITE,
57+
org.domain_aliases AS ORG_DOMAIN_ALIASES
58+
FROM FIVETRAN_INGEST.SFDC_CONNECTOR_PROD_PLATFORM.COMMUNITY__C c
59+
JOIN ANALYTICS.SILVER_DIM.COMMITTEE cm
60+
ON c.COLLABORATION_NAME__C = cm.COMMITTEE_ID
61+
INNER JOIN cdp_matched_segments cms
62+
ON cms.slug = cm.PROJECT_SLUG
63+
AND cms.sourceId = cm.PROJECT_ID
64+
LEFT JOIN ANALYTICS.SILVER_DIM.USERS su
65+
ON LOWER(c.CONTACTEMAIL__C) = LOWER(su.EMAIL)
66+
LEFT JOIN ANALYTICS.BRONZE_FIVETRAN_SALESFORCE_B2B.USERS bu
67+
ON c.CREATEDBYID = bu.USER_ID
68+
LEFT JOIN org_accounts org
69+
ON c.ACCOUNT__C = org.account_id
70+
WHERE c.LASTMODIFIEDDATE IS NOT NULL`
71+
72+
// Limit to a single project in non-prod to avoid exporting all project data
73+
if (!IS_PROD_ENV) {
74+
select += ` AND cm.PROJECT_SLUG = 'ccc'`
75+
}
76+
77+
const dedup = `
78+
QUALIFY ROW_NUMBER() OVER (PARTITION BY c.SFID ORDER BY org.website DESC) = 1`
79+
80+
if (!sinceTimestamp) {
81+
return `
82+
WITH ${ORG_ACCOUNTS},
83+
${CDP_MATCHED_SEGMENTS}
84+
${select}
85+
${dedup}`.trim()
86+
}
87+
88+
return `
89+
WITH ${ORG_ACCOUNTS},
90+
${CDP_MATCHED_SEGMENTS},
91+
new_cdp_segments AS (
92+
SELECT DISTINCT
93+
s.SOURCE_ID AS sourceId,
94+
s.slug
95+
FROM ANALYTICS.BRONZE_KAFKA_CROWD_DEV.SEGMENTS s
96+
WHERE s.CREATED_TS >= '${sinceTimestamp}'
97+
AND s.PARENT_SLUG IS NOT NULL
98+
AND s.GRANDPARENTS_SLUG IS NOT NULL
99+
AND s.SOURCE_ID IS NOT NULL
100+
)
101+
102+
-- Updated committee memberships since last export
103+
${select}
104+
AND c.LASTMODIFIEDDATE > '${sinceTimestamp}'
105+
${dedup}
106+
107+
UNION
108+
109+
-- All committee memberships in newly created segments
110+
${select}
111+
AND EXISTS (
112+
SELECT 1 FROM new_cdp_segments ncs
113+
WHERE ncs.slug = cms.slug AND ncs.sourceId = cms.sourceId
114+
)
115+
${dedup}`.trim()
116+
}

‎services/apps/snowflake_connectors/src/integrations/committees/committees/transformer.ts‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
import { COMMITTEES_GRID, CommitteesActivityType } from '@crowd/integrations'
2+
import { getServiceChildLogger } from '@crowd/logging'
3+
import {
4+
IActivityData,
5+
IMemberData,
6+
IOrganizationIdentity,
7+
MemberIdentityType,
8+
OrganizationIdentityType,
9+
OrganizationSource,
10+
PlatformType,
11+
} from '@crowd/types'
12+
13+
import { TransformedActivity, TransformerBase } from '../../../core/transformerBase'
14+
15+
const log = getServiceChildLogger('committeesCommitteesTransformer')
16+
17+
export class CommitteesCommitteesTransformer extends TransformerBase {
18+
readonly platform = PlatformType.COMMITTEES
19+
20+
transformRow(row: Record<string, unknown>): TransformedActivity | null {
21+
const email = (row.CONTACTEMAIL__C as string | null)?.trim() || null
22+
if (!email) {
23+
log.warn(
24+
{ sfid: row.SFID, committeeId: row.COMMITTEE_ID, rawEmail: row.CONTACTEMAIL__C },
25+
'Skipping row: missing email',
26+
)
27+
return null
28+
}
29+
30+
const committeeId = (row.COMMITTEE_ID as string).trim()
31+
const fivetranDeleted = row.FIVETRAN_DELETED as boolean
32+
const lfUsername = (row.LF_USERNAME as string | null)?.trim() || null
33+
const suFullName = (row.SU_FULL_NAME as string | null)?.trim() || null
34+
const suFirstName = (row.SU_FIRST_NAME as string | null)?.trim() || null
35+
const suLastName = (row.SU_LAST_NAME as string | null)?.trim() || null
36+
37+
const displayName =
38+
suFullName ||
39+
(suFirstName && suLastName ? `${suFirstName} ${suLastName}` : suFirstName || suLastName) ||
40+
email.split('@')[0]
41+
42+
const type = fivetranDeleted
43+
? CommitteesActivityType.REMOVED_FROM_COMMITTEE
44+
: CommitteesActivityType.ADDED_TO_COMMITTEE
45+
46+
const identities: IMemberData['identities'] = []
47+
48+
if (lfUsername) {
49+
identities.push(
50+
{
51+
platform: PlatformType.COMMITTEES,
52+
value: email,
53+
type: MemberIdentityType.EMAIL,
54+
verified: true,
55+
verifiedBy: PlatformType.COMMITTEES,
56+
},
57+
{
58+
platform: PlatformType.COMMITTEES,
59+
value: lfUsername,
60+
type: MemberIdentityType.USERNAME,
61+
verified: true,
62+
verifiedBy: PlatformType.COMMITTEES,
63+
},
64+
{
65+
platform: PlatformType.LFID,
66+
value: lfUsername,
67+
type: MemberIdentityType.USERNAME,
68+
verified: true,
69+
verifiedBy: PlatformType.COMMITTEES,
70+
},
71+
)
72+
}
73+
74+
if (!lfUsername) {
75+
identities.push({
76+
platform: PlatformType.COMMITTEES,
77+
value: email,
78+
type: MemberIdentityType.USERNAME,
79+
verified: true,
80+
verifiedBy: PlatformType.COMMITTEES,
81+
})
82+
}
83+
84+
const activity: IActivityData = {
85+
type,
86+
platform: PlatformType.COMMITTEES,
87+
timestamp: (row.LASTMODIFIEDDATE as string | null) || null,
88+
score: COMMITTEES_GRID[type].score,
89+
sourceId: committeeId,
90+
sourceParentId: null,
91+
member: {
92+
displayName,
93+
identities,
94+
organizations: this.buildOrganizations(row),
95+
},
96+
attributes: {
97+
committeeId: (row.COLLABORATION_NAME__C as string | null) || null,
98+
committeeName: (row.COMMITTEE_NAME as string | null) || null,
99+
role: (row.ROLE__C as string | null) || null,
100+
projectId: (row.PROJECT_ID as string | null) || null,
101+
projectName: (row.PROJECT_NAME as string | null) || null,
102+
organizationId: (row.ACCOUNT__C as string | null) || null,
103+
organizationName: (row.ACCOUNT_NAME as string | null) || null,
104+
member: {
105+
userId: (row.PRIMARY_SOURCE_USER_ID as string | null) || null,
106+
firstName: (row.SU_FIRST_NAME as string | null) || null,
107+
lastName: (row.SU_LAST_NAME as string | null) || null,
108+
email,
109+
},
110+
actor: {
111+
userId: (row.CREATEDBYID as string | null) || null,
112+
firstName: (row.BU_FIRST_NAME as string | null) || null,
113+
lastName: (row.BU_LAST_NAME as string | null) || null,
114+
email: (row.BU_EMAIL as string | null) || null,
115+
},
116+
activityDate: (row.CREATEDDATE as string | null) || null,
117+
},
118+
}
119+
120+
const segmentSlug = (row.PROJECT_SLUG as string | null)?.trim() || null
121+
const segmentSourceId = (row.PROJECT_ID as string | null)?.trim() || null
122+
123+
if (!segmentSlug || !segmentSourceId) {
124+
log.warn(
125+
{ sfid: row.SFID, committeeId, segmentSlug, segmentSourceId },
126+
'Skipping row: missing segment slug or sourceId',
127+
)
128+
return null
129+
}
130+
131+
return { activity, segment: { slug: segmentSlug, sourceId: segmentSourceId } }
132+
}
133+
134+
private buildOrganizations(row: Record<string, unknown>): IActivityData['member']['organizations'] {
135+
const website = (row.ORG_WEBSITE as string | null)?.trim() || null
136+
const domainAliases = (row.ORG_DOMAIN_ALIASES as string | null)?.trim() || null
137+
138+
if (!website && !domainAliases) {
139+
return undefined
140+
}
141+
142+
const displayName = (row.ACCOUNT_NAME as string | null)?.trim() || website
143+
144+
if (this.isIndividualNoAccount(displayName)) {
145+
return [
146+
{
147+
displayName,
148+
source: OrganizationSource.COMMITTEES,
149+
identities: website
150+
? [
151+
{
152+
platform: PlatformType.COMMITTEES,
153+
value: website,
154+
type: OrganizationIdentityType.PRIMARY_DOMAIN,
155+
verified: true,
156+
},
157+
]
158+
: [],
159+
},
160+
]
161+
}
162+
163+
const identities: IOrganizationIdentity[] = []
164+
165+
if (website) {
166+
identities.push({
167+
platform: PlatformType.COMMITTEES,
168+
value: website,
169+
type: OrganizationIdentityType.PRIMARY_DOMAIN,
170+
verified: true,
171+
})
172+
}
173+
174+
if (domainAliases) {
175+
for (const alias of domainAliases.split(',')) {
176+
const trimmed = alias.trim()
177+
if (trimmed) {
178+
identities.push({
179+
platform: PlatformType.COMMITTEES,
180+
value: trimmed,
181+
type: OrganizationIdentityType.ALTERNATIVE_DOMAIN,
182+
verified: true,
183+
})
184+
}
185+
}
186+
}
187+
188+
return [
189+
{
190+
displayName,
191+
source: OrganizationSource.COMMITTEES,
192+
identities,
193+
},
194+
]
195+
}
196+
}

‎services/apps/snowflake_connectors/src/integrations/index.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
*/
77
import { PlatformType } from '@crowd/types'
88

9+
import { buildSourceQuery as committeesCommitteesBuildQuery } from './committees/committees/buildSourceQuery'
10+
import { CommitteesCommitteesTransformer } from './committees/committees/transformer'
911
import { buildSourceQuery as cventBuildSourceQuery } from './cvent/event-registrations/buildSourceQuery'
1012
import { CventTransformer } from './cvent/event-registrations/transformer'
1113
import { buildSourceQuery as tncCertificatesBuildQuery } from './tnc/certificates/buildSourceQuery'
@@ -20,6 +22,15 @@ export type { BuildSourceQuery, DataSource, PlatformDefinition } from './types'
2022
export { DataSourceName } from './types'
2123

2224
const supported: Partial<Record<PlatformType, PlatformDefinition>> = {
25+
[PlatformType.COMMITTEES]: {
26+
sources: [
27+
{
28+
name: DataSourceName.COMMITTEES_COMMITTEES,
29+
buildSourceQuery: committeesCommitteesBuildQuery,
30+
transformer: new CommitteesCommitteesTransformer(),
31+
},
32+
],
33+
},
2334
[PlatformType.CVENT]: {
2435
sources: [
2536
{

‎services/apps/snowflake_connectors/src/integrations/types.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ export enum DataSourceName {
88
TNC_ENROLLMENTS = 'enrollments',
99
TNC_CERTIFICATES = 'certificates',
1010
TNC_COURSES = 'courses',
11+
COMMITTEES_COMMITTEES = 'committees',
1112
}
1213

1314
export interface DataSource {

‎services/libs/data-access-layer/src/organizations/attributesConfig.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,7 @@ export const ORG_DB_ATTRIBUTE_SOURCE_PRIORITY = [
233233
OrganizationAttributeSource.ENRICHMENT_PEOPLEDATALABS,
234234
OrganizationAttributeSource.CVENT,
235235
OrganizationAttributeSource.TNC,
236+
OrganizationAttributeSource.COMMITTEES,
236237
// legacy - keeping this for backward compatibility
237238
OrganizationAttributeSource.ENRICHMENT,
238239
OrganizationAttributeSource.GITHUB,

‎services/libs/integrations/src/integrations/committees/types.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
import { IActivityScoringGrid } from '@crowd/types'
2+
3+
export enum CommitteesActivityType {
4+
ADDED_TO_COMMITTEE = 'added-to-committee',
5+
REMOVED_FROM_COMMITTEE = 'removed-from-committee',
6+
}
7+
8+
export const COMMITTEES_GRID: Record<CommitteesActivityType, IActivityScoringGrid> = {
9+
[CommitteesActivityType.ADDED_TO_COMMITTEE]: { score: 1 },
10+
[CommitteesActivityType.REMOVED_FROM_COMMITTEE]: { score: 1 },
11+
}

‎services/libs/integrations/src/integrations/index.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,4 +52,6 @@ export * from './cvent/types'
5252

5353
export * from './tnc/types'
5454

55+
export * from './committees/types'
56+
5557
export * from './activityDisplayService'

‎services/libs/types/src/enums/organizations.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ export enum OrganizationSource {
1414
UI = 'ui',
1515
CVENT = 'cvent',
1616
TNC = 'tnc',
17+
COMMITTEES = 'committees',
1718
}
1819

1920
export enum OrganizationMergeSuggestionType {
@@ -42,6 +43,7 @@ export enum OrganizationAttributeSource {
4243
ENRICHMENT_PEOPLEDATALABS = 'enrichment-peopledatalabs',
4344
CVENT = 'cvent',
4445
TNC = 'tnc',
46+
COMMITTEES = 'committees',
4547
// legacy - keeping this for backward compatibility
4648
ENRICHMENT = 'enrichment',
4749
GITHUB = 'github',

0 commit comments

Comments
 (0)