Skip to content

Commit 789faf8

Browse files
elayiraelayira
authored
Organization enrichment (#825)
1 parent f1ae82c commit 789faf8

33 files changed

Lines changed: 876 additions & 5 deletions

‎backend/.env.dist.local‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,9 @@ CROWD_COHERE_API_KEY=
139139
CROWD_ENRICHMENT_URL=
140140
CROWD_ENRICHMENT_API_KEY=
141141

142+
# PDL Organization Enrichment settings
143+
CROWD_ORGANIZATION_ENRICHMENT_API_KEY=
144+
142145
# EagleEye settings
143146
CROWD_EAGLE_EYE_URL=
144147
CROWD_EAGLE_EYE_API_KEY=

‎backend/config/custom-environment-variables.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,9 @@
145145
"url": "CROWD_ENRICHMENT_URL",
146146
"apiKey": "CROWD_ENRICHMENT_API_KEY"
147147
},
148+
"organizationEnrichment": {
149+
"apiKey": "CROWD_ORGANIZATION_ENRICHMENT_API_KEY"
150+
},
148151
"eagleEye": {
149152
"url": "CROWD_EAGLE_EYE_URL",
150153
"apiKey": "CROWD_EAGLE_EYE_API_KEY"

‎backend/config/default.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
"github": {},
4040
"stackexchange": {},
4141
"enrichment": {},
42+
"organizationEnrichment": {},
4243
"eagleEye": {},
4344
"unleash": {
4445
"db": {}

‎backend/package-lock.json‎

Lines changed: 102 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎backend/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@
101101
"passport-google-oauth": "2.0.0",
102102
"passport-google-oauth20": "^2.0.0",
103103
"passport-slack": "0.0.7",
104+
"peopledatalabs": "^5.0.3",
104105
"pg": "^8.7.3",
105106
"pm2": "^5.2.0",
106107
"redis": "^4.5.0",

‎backend/src/bin/jobs/index.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import mergeSuggestions from './mergeSuggestions'
1111
import refreshSampleData from './refreshSampleData'
1212
import cleanUp from './cleanUp'
1313
import checkStuckIntegrationRuns from './checkStuckIntegrationRuns'
14+
import enrichOrganizations from './organizationEnricher'
1415

1516
const jobs: CrowdJob[] = [
1617
weeklyAnalyticsEmailsCoordinator,
@@ -25,6 +26,7 @@ const jobs: CrowdJob[] = [
2526
refreshSampleData,
2627
cleanUp,
2728
checkStuckIntegrationRuns,
29+
enrichOrganizations,
2830
]
2931

3032
export default jobs

‎backend/src/bin/jobs/organizationEnricher.ts‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
import cronGenerator from 'cron-time-generator'
2+
import SequelizeRepository from '../../database/repositories/sequelizeRepository'
3+
import { CrowdJob } from '../../types/jobTypes'
4+
import { sendNodeWorkerMessage } from '../../serverless/utils/nodeWorkerSQS'
5+
import { NodeWorkerMessageBase } from '../../types/mq/nodeWorkerMessageBase'
6+
import { NodeWorkerMessageType } from '../../serverless/types/workerTypes'
7+
import TenantRepository from '../../database/repositories/tenantRepository'
8+
import { getServiceLogger } from '../../utils/logging'
9+
10+
const job: CrowdJob = {
11+
name: 'organization enricher',
12+
cronTime: cronGenerator.everyDay(),
13+
onTrigger: sendWorkerMessage,
14+
}
15+
16+
async function sendWorkerMessage() {
17+
const options = await SequelizeRepository.getDefaultIRepositoryOptions()
18+
const log = getServiceLogger()
19+
const tenants = await TenantRepository.getPayingTenantIds(options)
20+
log.info(tenants)
21+
for (const { id } of tenants) {
22+
const payload = {
23+
type: NodeWorkerMessageType.NODE_MICROSERVICE,
24+
service: 'enrich-organizations',
25+
tenantId: id,
26+
} as NodeWorkerMessageBase
27+
log.info({ payload }, 'enricher worker payload')
28+
await sendNodeWorkerMessage(id, payload)
29+
}
30+
}
31+
32+
export default job

‎backend/src/config/configTypes.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,10 @@ export interface EnrichmentConfiguration {
186186
apiKey: string
187187
}
188188

189+
export interface OrganizationEnrichmentConfiguration {
190+
apiKey: string
191+
}
192+
189193
export interface EagleEyeConfiguration {
190194
url: string
191195
apiKey: string

‎backend/src/config/index.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import {
3030
SampleDataConfiguration,
3131
IntegrationProcessingConfiguration,
3232
SlackNotifierConfiguration,
33+
OrganizationEnrichmentConfiguration,
3334
} from './configTypes'
3435

3536
// TODO-kube
@@ -228,6 +229,12 @@ export const ENRICHMENT_CONFIG: EnrichmentConfiguration = KUBE_MODE
228229
apiKey: process.env.ENRICHMENT_SECRET_KEY,
229230
}
230231

232+
export const ORGANIZATION_ENRICHMENT_CONFIG: OrganizationEnrichmentConfiguration = KUBE_MODE
233+
? config.get<OrganizationEnrichmentConfiguration>('organizationEnrichment')
234+
: {
235+
apiKey: process.env.ORGANIZATION_ENRICHMENT_SECRET_KEY,
236+
}
237+
231238
export const EAGLE_EYE_CONFIG: EagleEyeConfiguration = KUBE_MODE
232239
? config.get<EagleEyeConfiguration>('eagleEye')
233240
: {

‎backend/src/database/initializers/sample-data.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -882,7 +882,7 @@
882882
"country": {
883883
"enrichment": "USA"
884884
},
885-
"yearsofExperience": {
885+
"yearsOfExperience": {
886886
"enrichment": 25
887887
},
888888
"education": {
@@ -1042,7 +1042,7 @@
10421042
"country": {
10431043
"enrichment": "USA"
10441044
},
1045-
"yearsofExperience": {
1045+
"yearsOfExperience": {
10461046
"enrichment": 5
10471047
},
10481048
"education": {
@@ -1187,7 +1187,7 @@
11871187
"country": {
11881188
"enrichment": "USA"
11891189
},
1190-
"yearsofExperience": {
1190+
"yearsOfExperience": {
11911191
"enrichment": 10
11921192
},
11931193
"education": {
@@ -1333,7 +1333,7 @@
13331333
"country": {
13341334
"enrichment": "USA"
13351335
},
1336-
"yearsofExperience": {
1336+
"yearsOfExperience": {
13371337
"enrichment": 20
13381338
},
13391339
"education": {

0 commit comments

Comments
 (0)