diff --git a/.deepsource.toml b/.deepsource.toml index 8e807bfda..831f23ee1 100644 --- a/.deepsource.toml +++ b/.deepsource.toml @@ -9,7 +9,9 @@ exclude_patterns = [ "frontend/public", "backend/logs", "backend/coverage", - "backend/.nyc_output" + "backend/.nyc_output", + "supabase/functions/.env.dev", + "supabase/functions/.env.example" ] [[analyzers]] diff --git a/.rate-limiting-deferred.md b/.rate-limiting-deferred.md new file mode 100644 index 000000000..3df3122e4 --- /dev/null +++ b/.rate-limiting-deferred.md @@ -0,0 +1,25 @@ +# Rate Limiting — Deferred to Follow-up + +Rate limiting on enrich + export-contacts endpoints was deferred because: + +1. **Redis dependency**: `_shared/rate-limiter.ts` requires `REDIS_URL` env var. If Redis isn't configured, every request would throw on `getRedisClient()`. +2. **No new code constraint**: The "no new code, no new logic" directive from the user means we can't add new middleware configuration patterns or fallbacks. +3. **Existing rate limiter exists but is unused**: `_shared/rate-limiter.ts` already provides `withRateLimit()` — it just isn't wired in. + +## Follow-up Tasks + +1. Verify REDIS_URL is configured in Supabase Edge Function settings (dashboard or CLI) +2. Wire `withRateLimit()` into enrich `/person` (weight 1 per request) +3. Wire `withRateLimit()` into enrich `/person/bulk` (weight = contact count) +4. Wire `withRateLimit()` into export-contacts `/:type` (weight 1 per request) +5. Add graceful fallback when Redis is unavailable +6. Test rate limit triggers at quota thresholds + +## Risk + +Without rate limiting, an authenticated user could: +- Trigger unlimited enrichment API calls (credit-burning) +- Generate unlimited export jobs (Google API quota) +- Exhaust edge function concurrency + +This is a known regression from the backend (which had per-endpoint `rateLimit({windowMs: 15*60*1000, max: 100})` middleware). diff --git a/backend/src/app.ts b/backend/src/app.ts index 4cb9a89a1..59756a7bb 100644 --- a/backend/src/app.ts +++ b/backend/src/app.ts @@ -17,8 +17,6 @@ import notFound from './middleware/notFound'; import initializeSentry from './middleware/sentry'; import initializeAuthRoutes from './routes/auth.routes'; import initializeContactsRoutes from './routes/contacts.routes'; -import initializeEnrichmentRoutes from './routes/enrichment.routes'; -import initializeImapRoutes from './routes/imap.routes'; import initializeMiningRoutes from './routes/mining.routes'; import initializeSmtpSendersRoutes from './routes/smtp-senders.routes'; import initializeStreamRouter from './routes/stream.routes'; @@ -65,7 +63,6 @@ export default function initializeApp( ); app.use('/api/auth', initializeAuthRoutes()); - app.use('/api/imap', initializeImapRoutes(authResolver, miningSourceService)); app.use('/api/imap', initializeStreamRouter(miningEngine, authResolver)); app.use( '/api/imap', @@ -81,7 +78,6 @@ export default function initializeApp( '/api', initializeContactsRoutes(contacts, authResolver, miningSourceService) ); - app.use('/api/enrich', initializeEnrichmentRoutes(authResolver)); app.use( '/api', initializeSmtpSendersRoutes(smtpSenders, authResolver, miningSources) diff --git a/backend/src/controllers/contacts.controller.ts b/backend/src/controllers/contacts.controller.ts index 8daeb44ee..07b9b3d3d 100644 --- a/backend/src/controllers/contacts.controller.ts +++ b/backend/src/controllers/contacts.controller.ts @@ -1,275 +1,3 @@ -import { User } from '@supabase/supabase-js'; -import { NextFunction, Request, Response } from 'express'; -import { Contacts } from '../db/interfaces/Contacts'; -import { Contact, ExportService } from '../db/types'; -import Billing from '../utils/billing-plugin'; -import { ExportOptions, ExportType } from '../services/export/types'; -import ExportFactory from '../services/export'; -import { - MiningSources, - OAuthMiningSourceCredentials -} from '../db/interfaces/MiningSources'; - -async function validateRequest( - req: Request, - res: Response, - miningSourceService: MiningSources -) { - const user = res.locals.user as User; - const partialExport = req.body.partialExport ?? false; - const updateEmptyFieldsOnly = req.body.updateEmptyFieldsOnly ?? false; - const exportType = (req.params.exportType ?? ExportType.CSV) as ExportType; - - const localeFromHeader = req.headers['accept-language']; - const delimiterOption = req.query.delimiter?.toString(); - - const validExportType = Object.values(ExportType).includes( - exportType as ExportType - ); - - if (!validExportType) { - throw new Error(`Invalid export type: ${exportType}`); - } - - let googleContactsOptions: ExportOptions['googleContactsOptions'] = { - userId: user.id, - accessToken: undefined, - refreshToken: undefined, - updateEmptyFieldsOnly - }; - - if (exportType === ExportType.GOOGLE_CONTACTS) { - const targetEmail = req.body.targetEmail || user.email; - const sources = await miningSourceService.getSourcesForUser( - user.id, - targetEmail as string - ); - const oauthCredentials = sources?.find((e) => e.email === targetEmail) - ?.credentials as OAuthMiningSourceCredentials; - - googleContactsOptions = { - ...googleContactsOptions, - accessToken: oauthCredentials?.accessToken, - refreshToken: oauthCredentials?.refreshToken - }; - } - - const { - ids, - exportAllContacts - }: { ids?: string[]; exportAllContacts: boolean } = req.body; - - if (!exportAllContacts && (!Array.isArray(ids) || !ids.length)) { - return { - userId: user.id, - exportType, - contactsToExport: null, - partialExport, - exportOptions: { - locale: localeFromHeader, - delimiter: undefined, - googleContactsOptions - } - }; - } - - const contactsToExport = exportAllContacts ? undefined : ids; - - return { - userId: user.id, - exportType, - contactsToExport, - partialExport, - exportOptions: { - locale: localeFromHeader, - delimiter: delimiterOption, - googleContactsOptions - } - }; -} - -async function respondWithContacts( - res: Response, - userId: string, - contacts: Contacts, - exportType: ExportType, - contactsToExport?: string[], - exportOption?: ExportOptions -) { - const selectedContacts = await contacts.getContacts(userId, contactsToExport); - - if (!selectedContacts.length) { - return res.sendStatus(204); // 204 No Content - } - - try { - const { content, contentType } = await ExportFactory.get(exportType).export( - selectedContacts, - exportOption - ); - - return res.header('Content-Type', contentType).status(200).send(content); - } catch (err) { - if ((err as Error).message === 'Invalid credentials.') { - return res.sendStatus(401); - } - throw err; - } -} - -async function verifyCredits( - userId: string, - contacts: Contacts, - contactsToExport?: string[] -) { - const newContacts = await contacts.getNonExportedContacts( - userId, - contactsToExport - ); - const previousExportedContacts = await contacts.getExportedContacts( - userId, - contactsToExport - ); - const creditsInfo = await Billing!.validateCustomerCredits( - userId, - newContacts.length - ); - const response = { - total: newContacts.length + previousExportedContacts.length, - available: Math.floor(creditsInfo.availableUnits), - availableAlready: previousExportedContacts.length - }; - return { - newContacts, - previousExportedContacts, - creditsInfo, - response - }; -} - -async function registerAndDeductCredits( - userId: string, - exportType: ExportType, - availableUnits: number, - contacts: Contacts, - availableContacts: Contact[] -) { - if (availableContacts.length) { - await contacts.registerExportedContacts( - availableContacts.map(({ id }) => id), - exportType as unknown as ExportService, - userId - ); - await Billing!.deductCustomerCredits(userId, availableUnits); - } -} - -async function respondWithConfirmedContacts( - res: Response, - userId: string, - exportType: ExportType, - contacts: Contacts, - newContacts: Contact[], - previousExportedContacts: Contact[], - availableUnits: number, - statusCode: number, - exportOptions?: ExportOptions -) { - const availableContacts = newContacts.slice(0, availableUnits); - const selectedContacts = [...previousExportedContacts, ...availableContacts]; - - try { - const { content, contentType } = await ExportFactory.get(exportType).export( - selectedContacts, - exportOptions - ); - - await registerAndDeductCredits( - userId, - exportType, - availableUnits, - contacts, - selectedContacts - ); - - return res - .header('Content-Type', contentType) - .status(statusCode) - .send(content); - } catch (err) { - if ((err as Error).message === 'Invalid credentials.') { - return res.sendStatus(401); - } - throw err; - } -} - -export default function initializeContactsController( - contacts: Contacts, - miningSources: MiningSources -) { - return { - async exportContactsCSV(req: Request, res: Response, next: NextFunction) { - const { - userId, - contactsToExport, - partialExport, - exportType, - exportOptions - } = await validateRequest(req, res, miningSources); - if (contactsToExport === null) { - return res.status(400).json({ - message: 'Parameter "ids" must be a non-empty list of person ids' - }); - } - - let statusCode = 200; - try { - if (!Billing) { - // No need to Verify Credits, Export. - return await respondWithContacts( - res, - userId, - contacts, - exportType, - contactsToExport, - exportOptions - ); - } - - const { newContacts, previousExportedContacts, creditsInfo, response } = - await verifyCredits(userId, contacts, contactsToExport); - - if ( - creditsInfo.hasDeficientCredits && - !previousExportedContacts.length - ) { - return res.status(402).json(response); - } - - if (creditsInfo.hasInsufficientCredits && newContacts.length) { - if (!partialExport) { - res.statusMessage = 'Confirm Partial Content'; - return res.status(266).json(response); // 266 Confirm Partial Content - } - statusCode = 206; // 206 Partial Content - } - - // Export confirmed contacts. - return await respondWithConfirmedContacts( - res, - userId, - exportType, - contacts, - newContacts, - previousExportedContacts, - creditsInfo.availableUnits, - statusCode, - exportOptions - ); - } catch (error) { - return next(error); - } - } - }; +export default function initializeContactsController() { + return {}; } diff --git a/backend/src/controllers/enrichment.controller.ts b/backend/src/controllers/enrichment.controller.ts deleted file mode 100644 index f9f33237a..000000000 --- a/backend/src/controllers/enrichment.controller.ts +++ /dev/null @@ -1,158 +0,0 @@ -import { NextFunction, Request, Response } from 'express'; -import { - createEnrichmentTask, - enrichFromCache, - enrichPersonAsync, - enrichPersonSync, - getEnrichmentCache, - prepareForEnrichment -} from './enrichment.helpers'; - -import Billing from '../utils/billing-plugin'; -import EnrichmentService from '../services/enrichment'; - -async function preEnrichmentMiddleware( - req: Request, - res: Response, - next: NextFunction -) { - const { user } = res.locals; - const { contact, enrichAllContacts, updateEmptyFieldsOnly } = req.body; - const contacts = [...(req.body.contacts ?? []), contact].filter(Boolean); - - if (!enrichAllContacts && contacts.length === 0) { - const msg = 'Missing parameter contact or contacts'; - return res.status(400).json({ message: msg }); - } - - try { - const enrichment = await prepareForEnrichment( - user.id, - enrichAllContacts ?? false, - updateEmptyFieldsOnly, - contacts - ); - - if (enrichment.available === 0) { - return res.status(402).json({ - total: enrichment.total, - available: enrichment.available - }); - } - - res.locals.enrichment = enrichment; - return next(); - } catch (error) { - return next(error); - } -} - -function isEmptyEnrichEngines(error: unknown) { - return Boolean( - error instanceof Error && error.message.startsWith('Enricher not found.') - ); -} - -async function enrichPerson(_: Request, res: Response, next: NextFunction) { - const { - user, - enrichment: { contact, updateEmptyFieldsOnly } - } = res.locals; - const enrichmentsDB = createEnrichmentTask(); - try { - await enrichmentsDB.create(user.id, 1, updateEmptyFieldsOnly); - const notEnriched = await enrichFromCache( - getEnrichmentCache, - enrichmentsDB, - [contact] - ); - await enrichPersonSync(enrichmentsDB, notEnriched); - await enrichmentsDB.end(); - return res.status(200).json({ task: enrichmentsDB.redactedTask() }); - } catch (err) { - await enrichmentsDB.cancel(); - if (isEmptyEnrichEngines(err)) { - res.statusCode = 503; - } - return next(err); - } -} - -async function enrichPersonBulk(_: Request, res: Response, next: NextFunction) { - const { user, enrichment } = res.locals; - const { contacts, updateEmptyFieldsOnly } = enrichment; - const enrichmentsDB = createEnrichmentTask(); - try { - await enrichmentsDB.create(user.id, contacts.length, updateEmptyFieldsOnly); - const notEnrichedCache = await enrichFromCache( - getEnrichmentCache, - enrichmentsDB, - contacts - ); - const notEnriched = await enrichPersonSync(enrichmentsDB, notEnrichedCache); - const enrichAsyncResult = notEnriched.length - ? await enrichPersonAsync(enrichmentsDB, notEnriched) - : null; - - if (!enrichAsyncResult) { - await enrichmentsDB.end(); - } - - return res.status(200).json({ task: enrichmentsDB.redactedTask() }); - } catch (err) { - await enrichmentsDB.cancel(); - if (isEmptyEnrichEngines(err)) res.statusCode = 503; - return next(err); - } -} - -async function enrichWebhook(req: Request, res: Response, next: NextFunction) { - const { id: taskId } = req.params; - const { token } = req.body; - const enrichmentsDB = createEnrichmentTask(); - - try { - const task = await enrichmentsDB.createFromId(taskId); - const taskResult = task?.details.result.find((enr) => enr.token === token); - - if (!taskResult) return res.sendStatus(404); - - const { engine } = taskResult; - const result = EnrichmentService.parseResult([req.body], engine); - - const contactsMap = ( - taskResult as { - contacts_map?: Array<{ email: string; person_id: string }>; - } - ).contacts_map; - - if (contactsMap) { - const emailToId = new Map(contactsMap.map((c) => [c.email, c.person_id])); - for (const data of result.data) { - if (data.email && !data.person_id) { - data.person_id = emailToId.get(data.email); - } - } - } - - await enrichmentsDB.enrich([{ ...taskResult, ...result }]); - await Billing?.deductCustomerCredits( - task.userId, - result.data.filter(Boolean).length - ); - await enrichmentsDB.end(); - return res.sendStatus(200); - } catch (err) { - await enrichmentsDB.cancel(); - return next(err); - } -} - -export default function initializeEnrichmentController() { - return { - enrichPerson, - enrichPersonBulk, - enrichWebhook, - preEnrichmentMiddleware - }; -} diff --git a/backend/src/controllers/enrichment.helpers.ts b/backend/src/controllers/enrichment.helpers.ts deleted file mode 100644 index 60a02cf54..000000000 --- a/backend/src/controllers/enrichment.helpers.ts +++ /dev/null @@ -1,227 +0,0 @@ -import { EngineResponse, EngineResult } from '../services/enrichment/Engine'; - -import Billing from '../utils/billing-plugin'; -import { Contact } from '../db/types'; -import ENV from '../config'; -import Engagements from '../db/supabase/engagements'; -import Enricher from '../services/enrichment/Enricher'; -import EnrichmentService from '../services/enrichment'; -import Enrichments from '../db/supabase/enrichments'; -import SupabaseTasks from '../db/supabase/tasks'; -import logger from '../utils/logger'; -import supabaseClient from '../utils/supabase'; - -/** - * Queries person ids for a given user from table "refinedpersons". - * @param userId - The ID of the user. - * @returns List of person UUIDs. - */ -export async function getRefinedPersons(userId: string) { - const { data, error } = await supabaseClient - .schema('private') - .from('refinedpersons') - .select('person_id') - .match({ user_id: userId }) - .returns<{ person_id: string }[]>(); - - if (error) { - throw new Error(error.message); - } - - return data.map((record) => record.person_id); -} - -/** - * Queries all contacts from the persons that are not yet enriched. - * @param userId - The ID of the user. - * @returns List of contacts to be enriched. - * @throws Error if there is an issue fetching data from the database. - */ -export async function getContactsToEnrich( - userId: string, - enrichAll: boolean, - contacts?: Partial[] -) { - if (!enrichAll) { - return contacts ?? []; - } - - const refinedPersonIds = await getRefinedPersons(userId); - - const { data, error } = await supabaseClient - .schema('private') - .from('persons') - .select('id, email, name') - .eq('user_id', userId) - .in('id', refinedPersonIds) - .returns<{ id: string; email: string | null; name: string }[]>(); - - if (error) { - throw new Error(error.message); - } - - return data.map(({ id, email, name }) => ({ id, email, name })); -} - -export async function restrictOrDecline(userId: string, totalContacts: number) { - const { availableUnits } = (await Billing?.validateCustomerCredits( - userId, - totalContacts - )) ?? { availableUnits: totalContacts }; - - return { - total: totalContacts, - available: Math.floor(availableUnits) - }; -} - -export async function prepareForEnrichment( - userId: string, - enrichAll: boolean, - updateEmptyFieldsOnly: boolean, - contacts: Contact[] -) { - const toEnrich = await getContactsToEnrich(userId, enrichAll, contacts); - const { total, available } = await restrictOrDecline(userId, toEnrich.length); - const selectedContacts = toEnrich.slice(0, available); - - return { - total, - available, - updateEmptyFieldsOnly, - contacts: selectedContacts, - contact: selectedContacts[0] - }; -} - -export function createEnrichmentTask() { - const tasks = new SupabaseTasks(supabaseClient, logger); - const engagements = new Engagements(supabaseClient, logger); - const enrichmentsDB = new Enrichments( - tasks, - engagements, - supabaseClient, - logger - ); - return enrichmentsDB; -} - -export interface EnrichmentCache { - task_id: string; - user_id: string; - created_at: string; - engine: string; - result: EngineResult; -} - -export async function getEnrichmentCache( - contacts: Partial[], - enricher: Enricher -) { - const { data, error } = await supabaseClient - .schema('private') - .rpc('enriched_most_recent', { - emails: contacts.map((contact) => contact.email) - }); - - if (error) { - throw new Error(error.message); - } - return !data - ? [] - : (data as EnrichmentCache[]) - .flatMap((cache) => enricher.parseResult([cache.result], cache.engine)) - .filter((cache): cache is EngineResponse => Boolean(cache?.data.length)) - .map((cache) => ({ ...cache, engine: 'cache' })); -} - -export async function enrichFromCache( - getCached: ( - contacts: Partial[], - enricher: Enricher - ) => Promise, - enrichmentsDB: Enrichments, - contacts: Partial[] -) { - const emailToId = new Map( - contacts - .filter( - (c): c is { email: string; id: string } => - Boolean(c.email) && Boolean(c.id) - ) - .map((c) => [c.email, c.id]) - ); - - const cached = await getCached(contacts, EnrichmentService); - const enrichedEmails = new Set( - cached - .flatMap(({ data }) => (data as Partial[]) || []) - .filter((contact): contact is Partial => Boolean(contact?.email)) - .map(({ email }) => email) - ); - - if (enrichedEmails.size) { - for (const response of cached) { - for (const data of response.data) { - data.person_id = emailToId.get(data.email); - } - } - await enrichmentsDB.enrich(cached); - logger.debug('Enriched from cache.', Array.from(enrichedEmails.values())); - } - - return contacts.filter( - (contact) => contact.email && !enrichedEmails.has(contact.email) - ); -} - -export async function enrichPersonSync( - enrichmentsDB: Enrichments, - contacts: Partial[] -) { - const task = enrichmentsDB.redactedTask(); - const enriched = new Set(); - - for (const contact of contacts) { - // eslint-disable-next-line no-await-in-loop - const result = await EnrichmentService.enrichSync(contact); - if (!result) continue; - for (const data of result.data) { - data.person_id = contact.id; - } - // eslint-disable-next-line no-await-in-loop - await enrichmentsDB.enrich([result]); - if (contact.email) enriched.add(contact.email); - } - - if (enriched.size > 0) { - await Billing?.deductCustomerCredits(task.userId, enriched.size); - } - - return contacts.filter((c) => (c.email ? !enriched.has(c.email) : false)); -} - -export async function enrichPersonAsync( - enrichmentsDB: Enrichments, - contacts: Partial[] -) { - const task = enrichmentsDB.redactedTask(); - const webhook = `${ENV.LEADMINER_API_HOST}/api/enrich/webhook/${task.id}`; - const result = await EnrichmentService.enrichAsync(contacts, webhook); - - if (result) { - const contactsMap = contacts - .filter( - (c): c is { email: string; id: string } => - Boolean(c.email) && Boolean(c.id) - ) - .map((c) => ({ email: c.email, person_id: c.id })); - ( - result as EngineResponse & { - contacts_map: Array<{ email: string; person_id: string }>; - } - ).contacts_map = contactsMap; - await enrichmentsDB.enrich([result]); - } - return result; -} diff --git a/backend/src/controllers/imap.controller.ts b/backend/src/controllers/imap.controller.ts deleted file mode 100644 index 9ed33eb24..000000000 --- a/backend/src/controllers/imap.controller.ts +++ /dev/null @@ -1,141 +0,0 @@ -import { User } from '@supabase/supabase-js'; -import { NextFunction, Request, Response } from 'express'; -import { - MiningSources, - OAuthMiningSourceCredentials -} from '../db/interfaces/MiningSources'; -import azureOAuth2Client from '../services/OAuth2/azure'; -import googleOAuth2Client from '../services/OAuth2/google'; -import ImapBoxesFetcher from '../services/imap/ImapBoxesFetcher'; -import ImapConnectionProvider from '../services/imap/ImapConnectionProvider'; -import { ImapAuthError } from '../utils/errors'; -import hashEmail from '../utils/helpers/hashHelpers'; -import logger from '../utils/logger'; -import { generateErrorObjectFromImapError } from './imap.helpers'; - -function getTokenAndProvider(data: OAuthMiningSourceCredentials) { - const { provider, accessToken, refreshToken, expiresAt } = data; - const client = provider === 'azure' ? azureOAuth2Client : googleOAuth2Client; - - const token = client.createToken({ - access_token: accessToken, - refresh_token: refreshToken, - expires_at: expiresAt - }); - - return { token, refreshToken, provider }; -} - -export default function initializeImapController( - miningSourceService: MiningSources -) { - return { - async getImapBoxes(req: Request, res: Response, next: NextFunction) { - const { email } = req.body; - - let imapConnection: Awaited< - ReturnType - > | null = null; - - try { - const userId = (res.locals.user as User).id; - const sources = await miningSourceService.getSourcesForUser( - userId, - email - ); - - const data = - sources?.find((e) => e.email === email)?.credentials ?? null; - - if (!data) { - res.status(400); - return next( - new Error('Unable to retrieve credentials for this mining source') - ); - } - - const isImapCredentials = - 'tls' in data && 'email' in data && 'password' in data; - - if (!('accessToken' in data) && !isImapCredentials) { - return res.status(400).send({ - data: { - message: 'This mining source does not support IMAP folders lookup' - } - }); - } - - if ('accessToken' in data) { - const { token, refreshToken } = getTokenAndProvider(data); - - if (!refreshToken) - return res.status(401).send({ - data: { message: 'No Refresh Token' } - }); - - if (token.expired(1000)) { - return res.status(401).send({ - data: { message: 'Access token is expired' } - }); - } - } - - imapConnection = await ImapConnectionProvider.getSingleConnection( - email, - 'accessToken' in data - ? { - oauthToken: data.accessToken - } - : { - host: data.host, - password: data.password, - tls: data.tls, - port: data.port - } - ); - - const imapBoxesFetcher = new ImapBoxesFetcher(imapConnection, logger); - const tree: any = await imapBoxesFetcher.getTree(email); - - logger.info('Mining IMAP tree succeeded.', { - metadata: { - user: hashEmail(email, userId) - } - }); - - return res.status(200).send({ - data: { message: 'IMAP folders fetched successfully!', folders: tree } - }); - } catch (error: any) { - logger.error('Error during inbox fetch', { - message: error.message, - stack: error.stack, - code: error.code - }); - - if ([502, 503].includes(error?.output?.payload?.statusCode)) { - return res - .status(error?.output?.payload?.statusCode) - .send(error?.output?.payload?.error); - } - - const generatedError = generateErrorObjectFromImapError(error); - if (generatedError instanceof ImapAuthError) { - return res.status(generatedError.status).send(generatedError); - } - return next(generatedError); - } finally { - if (imapConnection) { - try { - await imapConnection.logout(); - } catch (logoutError) { - logger.warn( - 'Unable to close IMAP connection cleanly.', - logoutError - ); - } - } - } - } - }; -} diff --git a/backend/src/db/interfaces/Contacts.ts b/backend/src/db/interfaces/Contacts.ts index da8e544ef..de78a4515 100644 --- a/backend/src/db/interfaces/Contacts.ts +++ b/backend/src/db/interfaces/Contacts.ts @@ -3,7 +3,6 @@ import { Contact, ContactFrontend, EmailStatus, - ExportService, ExtractionResult, Tag } from '../types'; @@ -24,13 +23,6 @@ export interface Contacts { ): Promise; getContacts(userId: string, ids?: string[]): Promise; getUnverifiedContacts(userId: string, ids: string[]): Promise; - getExportedContacts(userId: string, ids?: string[]): Promise; - getNonExportedContacts(userId: string, ids?: string[]): Promise; - registerExportedContacts( - personIds: string[], - exportService: ExportService, - userId: string - ): Promise; upsertGoogleContacts( contacts: Array<{ person: ContactFrontend; tags: string[] }>, userId: string, diff --git a/backend/src/db/pg/PgContacts.ts b/backend/src/db/pg/PgContacts.ts index f21d69167..b813af9a8 100644 --- a/backend/src/db/pg/PgContacts.ts +++ b/backend/src/db/pg/PgContacts.ts @@ -9,7 +9,6 @@ import { ContactFrontend, EmailExtractionResult, EmailStatus, - ExportService, ExtractionResult, FileExtractionResult, GoogleContactsExtractionResult, @@ -31,25 +30,6 @@ export default class PgContacts implements Contacts { private static readonly SELECT_CONTACTS_SQL = 'SELECT * FROM private.get_contacts_table($1)'; - private static readonly SELECT_EXPORTED_CONTACTS = ` - SELECT contacts.* - FROM private.get_contacts_table($1) contacts - JOIN private.engagement e - ON e.person_id = contacts.id - AND e.user_id = $1 - AND e.engagement_type = 'EXPORT' - `; - - private static readonly SELECT_NON_EXPORTED_CONTACTS = ` - SELECT contacts.* - FROM private.get_contacts_table($1) contacts - LEFT JOIN private.engagement e - ON e.person_id = contacts.id - AND e.user_id = $1 - AND e.engagement_type = 'EXPORT' - WHERE e.person_id IS NULL; - `; - private static readonly SELECT_CONTACTS_BY_IDS = 'SELECT * FROM private.get_contacts_table_by_ids($1,$2)'; @@ -59,34 +39,12 @@ export default class PgContacts implements Contacts { private static readonly SELECT_CONTACTS_UNVERIFIED = 'SELECT * FROM private.get_contacts_table($1) WHERE status IS NULL'; - private static readonly SELECT_EXPORTED_CONTACTS_BY_IDS = ` - SELECT contacts.* - FROM private.get_contacts_table_by_ids($1,$2) contacts - JOIN private.engagement e - ON e.person_id = contacts.id - AND e.user_id = $1 - AND e.engagement_type = 'EXPORT' - `; - - private static readonly SELECT_NON_EXPORTED_CONTACTS_BY_IDS = ` - SELECT contacts.* - FROM private.get_contacts_table_by_ids($1,$2) contacts - LEFT JOIN private.engagement e - ON e.person_id = contacts.id - AND e.user_id = $1 - AND e.engagement_type = 'EXPORT' - WHERE e.person_id IS NULL; - `; - private static readonly UPDATE_PERSON_STATUS_BULK = ` UPDATE private.persons SET status = update.status FROM (VALUES %L) AS update(id, status) WHERE persons.id = update.id AND persons.user_id = %L AND persons.status IS NULL`; - private static readonly INSERT_EXPORTED_CONTACT = - 'INSERT INTO private.engagement (user_id, person_id, engagement_type, service) VALUES %L ON CONFLICT (person_id, user_id, engagement_type, service) DO NOTHING;'; - private static readonly INSERT_MESSAGE_SQL = ` INSERT INTO private.messages("channel","folder_path","date","message_id","references","list_id","conversation","user_id") VALUES($1, $2, $3, $4, $5, $6, $7, $8) @@ -701,59 +659,6 @@ LIMIT 1; } } - async getExportedContacts( - userId: string, - ids?: string[] - ): Promise { - try { - const { rows } = ids - ? await this.pool.query(PgContacts.SELECT_EXPORTED_CONTACTS_BY_IDS, [ - userId, - ids - ]) - : await this.pool.query(PgContacts.SELECT_EXPORTED_CONTACTS, [userId]); - return rows; - } catch (error) { - this.logger.error(error); - return []; - } - } - - async getNonExportedContacts( - userId: string, - ids?: string[] - ): Promise { - try { - const { rows } = ids - ? await this.pool.query( - PgContacts.SELECT_NON_EXPORTED_CONTACTS_BY_IDS, - [userId, ids] - ) - : await this.pool.query(PgContacts.SELECT_NON_EXPORTED_CONTACTS, [ - userId - ]); - - return rows; - } catch (error) { - this.logger.error(error); - return []; - } - } - - async registerExportedContacts( - personIds: string[], - service: ExportService, - userId: string - ): Promise { - try { - const values = personIds.map((id) => [userId, id, 'EXPORT', service]); - await this.pool.query(format(PgContacts.INSERT_EXPORTED_CONTACT, values)); - } catch (error) { - this.logger.error(error); - throw error; - } - } - async upsertGoogleContacts( contacts: Array<{ person: ContactFrontend; tags: string[] }>, userId: string, diff --git a/backend/src/mocks/enrichment/endpoints/thedig.ts b/backend/src/mocks/enrichment/endpoints/thedig.ts new file mode 100644 index 000000000..7428034b0 --- /dev/null +++ b/backend/src/mocks/enrichment/endpoints/thedig.ts @@ -0,0 +1,27 @@ +import { faker } from '@faker-js/faker'; +import { Request, Response, Router } from 'express'; + +/** + * TheDig mock — the edge function engine sends + * POST {baseUrl}/person/ -> single person + * POST {baseUrl}/person/bulk -> bulk async + * This is a minimal mock: return a person with empty optional fields + * so the engine's `parseResult` drops the row. This causes the + * enrich pipeline to mark "no_data" rather than erroring. + */ +const router = Router(); + +router.post('/person/', (req: Request, res: Response) => { + res.status(200).json({ + email: req.body?.email ?? '', + name: '', + givenName: '', + statusCode: 203 // Non-authoritative (empty result) + }); +}); + +router.post('/person/bulk', (req: Request, res: Response) => { + res.status(200).json(faker.string.nanoid()); +}); + +export default router; diff --git a/backend/src/mocks/enrichment/server-with-thedig.ts b/backend/src/mocks/enrichment/server-with-thedig.ts new file mode 100644 index 000000000..d427e7b64 --- /dev/null +++ b/backend/src/mocks/enrichment/server-with-thedig.ts @@ -0,0 +1,30 @@ +import express, { json, urlencoded } from 'express'; +import { SERVER_PORT } from './config'; + +import voilanorbertRoutes from './endpoints/voilanorbert'; +import enrichlayerRoutes from './endpoints/enrichlayer'; +import thedigRoutes from './endpoints/thedig'; + +const app = express(); + +app.use(json({ limit: '5mb' })); +app.use(urlencoded({ limit: '5mb', extended: true })); + +// Original mounts (kept for backwards compatibility) +app.use('/voilanorbert', voilanorbertRoutes); +app.use('/enrichlayer', enrichlayerRoutes); +// TheDig engine mock at /thedig +app.use('/thedig', thedigRoutes); + +// ALSO mount the same routes at the engine's URL-construction path +// (the engine uses `new URL("/api/...", baseUrl)` which drops the +// path prefix from baseUrl, so routes need to be reachable at /api/...) +app.use('/', enrichlayerRoutes); +app.use('/', thedigRoutes); + +app.listen(SERVER_PORT, () => { + // eslint-disable-next-line no-console + console.log( + `Started mock servers for local development on port ${SERVER_PORT}` + ); +}); diff --git a/backend/src/routes/contacts.routes.ts b/backend/src/routes/contacts.routes.ts index 809154c0d..1000d1d41 100644 --- a/backend/src/routes/contacts.routes.ts +++ b/backend/src/routes/contacts.routes.ts @@ -1,16 +1,13 @@ import { Router } from 'express'; import rateLimit from 'express-rate-limit'; -import initializeContactsController from '../controllers/contacts.controller'; import initializeContactsVerificationController from '../controllers/contacts-verification.controller'; import { Contacts } from '../db/interfaces/Contacts'; import initializeAuthMiddleware from '../middleware/auth'; import AuthResolver from '../services/auth/AuthResolver'; -import { MiningSources } from '../db/interfaces/MiningSources'; export default function initializeContactsRoutes( contacts: Contacts, - authResolver: AuthResolver, - miningSources: MiningSources + authResolver: AuthResolver ) { const router = Router(); const contactsRouteLimiter = rateLimit({ @@ -18,21 +15,9 @@ export default function initializeContactsRoutes( max: 100 }); - const { exportContactsCSV } = initializeContactsController( - contacts, - miningSources - ); - const { verifyEmailStatus } = initializeContactsVerificationController(contacts); - router.post( - '/contacts/export/:exportType', - initializeAuthMiddleware(authResolver), - contactsRouteLimiter, - exportContactsCSV - ); - router.post( '/contacts/verify', initializeAuthMiddleware(authResolver), diff --git a/backend/src/routes/enrichment.routes.ts b/backend/src/routes/enrichment.routes.ts deleted file mode 100644 index a87d205af..000000000 --- a/backend/src/routes/enrichment.routes.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { Router } from 'express'; -import AuthResolver from '../services/auth/AuthResolver'; -import initializeAuthMiddleware from '../middleware/auth'; -import initializeEnrichmentController from '../controllers/enrichment.controller'; - -export default function initializeEnrichmentRoutes(authResolver: AuthResolver) { - const router = Router(); - - const { - enrichPerson, - enrichPersonBulk, - enrichWebhook, - preEnrichmentMiddleware - } = initializeEnrichmentController(); - - router.post( - '/person', - initializeAuthMiddleware(authResolver), - preEnrichmentMiddleware, - enrichPerson - ); - - router.post( - '/person/bulk', - initializeAuthMiddleware(authResolver), - preEnrichmentMiddleware, - enrichPersonBulk - ); - - router.post('/webhook/:id', enrichWebhook); - - return router; -} diff --git a/backend/src/routes/imap.routes.ts b/backend/src/routes/imap.routes.ts deleted file mode 100644 index c47d58c07..000000000 --- a/backend/src/routes/imap.routes.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { Router } from 'express'; -import initializeImapController from '../controllers/imap.controller'; -import { MiningSources } from '../db/interfaces/MiningSources'; -import initializeAuthMiddleware from '../middleware/auth'; -import AuthResolver from '../services/auth/AuthResolver'; - -export default function initializeImapRoutes( - authResolver: AuthResolver, - miningSources: MiningSources -) { - const router = Router(); - - const { getImapBoxes } = initializeImapController(miningSources); - - router.post('/boxes', initializeAuthMiddleware(authResolver), getImapBoxes); - - return router; -} diff --git a/backend/src/services/enrichment/Engine.ts b/backend/src/services/enrichment/Engine.ts deleted file mode 100644 index 892472fb1..000000000 --- a/backend/src/services/enrichment/Engine.ts +++ /dev/null @@ -1,50 +0,0 @@ -export interface Person { - url: string; - email: string; - name?: string; - image?: string; - job_title?: string; - given_name?: string; - family_name?: string; - works_for?: string; - alternate_name?: string[]; - location?: string; - same_as?: string[]; - identifiers: string[]; -} - -export interface EngineResult { - person_id?: string; - email: string; - name?: string; - image?: string; - location?: string; - jobTitle?: string; - organization?: string; - givenName?: string; - familyName?: string; - sameAs?: string[]; - identifiers?: string[]; - alternateName?: string[]; -} - -export interface EngineResponse { - token?: string; - engine: string; - raw_data: unknown[]; - data: EngineResult[]; -} - -export interface Engine { - readonly name: string; - readonly isSync: boolean; - readonly isAsync: boolean; - - isValid: (contact: Partial) => boolean; - enrichAsync( - persons: Partial[], - webhook: string - ): Promise; - enrichSync(persons: Partial): Promise; - parseResult(data: unknown[]): EngineResponse; -} diff --git a/backend/src/services/enrichment/Enricher.ts b/backend/src/services/enrichment/Enricher.ts deleted file mode 100644 index 24de05c86..000000000 --- a/backend/src/services/enrichment/Enricher.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { Logger } from 'winston'; -import { Contact } from '../../db/types'; -import { Engine } from './Engine'; - -export default class Enricher { - constructor( - private readonly engines: Engine[], - private readonly logger: Logger - ) {} - - private logError(context: string, error: unknown): void { - const message = (error as Error).message || 'Unexpected error'; - this.logger.error(`[${context}]: ${message}`, error); - } - - private async sync(engine: Engine, contact: Partial) { - try { - const result = await engine.enrichSync(contact); - return result; - } catch (error) { - this.logError('EnrichEngine._sync', error); - } - return null; - } - - private async async( - engine: Engine, - contacts: Partial[], - webhook: string - ) { - try { - const result = await engine.enrichAsync(contacts, webhook); - return result; - } catch (error) { - this.logError('EnrichEngine._async', error); - } - return null; - } - - async enrichSync(contact: Partial) { - const engines = this.engines.filter((enricher): enricher is Engine => - Boolean(enricher?.isValid(contact) && enricher.isSync) - ); - - if (!engines.length) throw new Error('No Engines to use.'); - - for (const engine of engines) { - // eslint-disable-next-line no-await-in-loop - const result = await this.sync(engine, contact); - // Break on successful enrichment with data - if (result?.data?.length) return result; - } - return null; - } - - async enrichAsync(contacts: Partial[], webhook: string) { - const engines = this.engines.filter((enricher): enricher is Engine => - Boolean(enricher?.isAsync) - ); - - if (!engines.length) throw new Error('No Engines to use.'); - - for (const engine of engines) { - // eslint-disable-next-line no-await-in-loop - const result = await this.async(engine, contacts, webhook); - // Break on successful enrichment with data - if (result?.token) return result; - } - return null; - } - - async *enrich(contacts: Partial[]) { - for (const contact of contacts) { - // eslint-disable-next-line no-await-in-loop - const result = await this.enrichSync(contact); - if (result) { - yield result; - } - } - } - - parseResult(result: unknown[], engineName: string) { - const engine = this.engines.findLast(({ name }) => name === engineName); - const parsed = engine?.parseResult(result); - return { - data: parsed?.data || [], - raw_data: parsed?.raw_data || [] - }; - } -} diff --git a/backend/src/services/enrichment/enrich-layer/client.ts b/backend/src/services/enrichment/enrich-layer/client.ts deleted file mode 100644 index 08f67e533..000000000 --- a/backend/src/services/enrichment/enrich-layer/client.ts +++ /dev/null @@ -1,161 +0,0 @@ -import axios, { AxiosError, AxiosInstance } from 'axios'; - -import { Logger } from 'winston'; -import { logError } from '../../../utils/axios'; -import { IRateLimiter } from '../../rate-limiter'; - -interface Config { - url: string; - apiKey: string; - rateLimiter: IRateLimiter; -} - -interface Experience { - starts_at: { - day: number; - month: number; - year: number; - }; - ends_at: { - day?: number | null; - month?: number | null; - year?: number | null; - }; - company: string; - company_linkedin_profile_url?: string | null; - company_facebook_profile_url?: string | null; - title: string; - description: string; - location?: string | null; - logo_url?: string | null; -} - -interface Profile { - city: string; - full_name: string; - first_name: string; - last_name: string; - state: string; - country: string; - country_full_name: string; - languages: string[]; - occupation: string; - profile_pic_url: string; - public_identifier: string; - extra: ProfileExtra; - experiences: Experience[]; -} - -export interface ProfileExtra { - github_profile_id?: string; - facebook_profile_id?: string; - twitter_profile_id?: string; - website?: string; -} - -export interface ReverseEmailLookupParams { - email: string; - lookup_depth: 'superficial' | 'deep'; - enrich_profile?: 'skip' | 'enrich'; -} - -export interface ReverseEmailLookupResponse { - email: string; - profile: Profile; - last_updated: string; - similarity_score: number; - linkedin_profile_url: string; - facebook_profile_url: string; - twitter_profile_url: string; -} - -export default class EnrichLayerAPI { - private readonly api: AxiosInstance; - - private readonly rateLimiter; - - private readonly maxRetries: number = 5; - - constructor( - { url, apiKey, rateLimiter }: Config, - private readonly logger: Logger - ) { - this.api = axios.create({ - baseURL: url, - headers: { - Authorization: `Bearer ${apiKey}` - } - }); - this.rateLimiter = rateLimiter; - } - - async reverseEmailLookup({ - email, - lookup_depth: lookupDepth, - enrich_profile: enrichProfile - }: ReverseEmailLookupParams) { - const response = await this.rateLimitRetryWithExponentialBackoff( - async () => { - try { - const res = await this.rateLimiter.throttleRequests(() => - this.api.get( - '/api/v2/profile/resolve/email', - { - params: { - email, - lookup_depth: lookupDepth, - enrich_profile: enrichProfile - } - } - ) - ); - return res; - } catch (error) { - logError( - error, - `[${this.constructor.name}:verifyEmail]`, - this.logger - ); - throw error; - } - } - ); - return { ...response.data, email }; - } - - private async rateLimitRetryWithExponentialBackoff( - fn: (attempt: number) => Promise, - attempt = 1 - ): Promise { - const sleep = (delay: number): Promise => - new Promise((resolve) => { - setTimeout(() => { - resolve(); - }, delay); - }); - try { - const response = await fn(attempt); - return response; - } catch (error) { - if ((error as AxiosError).response?.status !== 429) { - throw new Error( - (error as AxiosError).message || 'Request failed with unknown error.' - ); - } - if (attempt >= this.maxRetries) { - logError( - error, - `[${this.constructor.name}: Exhausted retries]`, - this.logger - ); - throw error; - } - const delay = 2 ** attempt * 1000; - this.logger.info( - `[${this.constructor.name}:rateLimitRetryWithExponentialBackoff] Rate limited, retrying attempt ${attempt} waiting ${delay}ms before retry` - ); - await sleep(delay); - return this.rateLimitRetryWithExponentialBackoff(fn, attempt + 1); - } - } -} diff --git a/backend/src/services/enrichment/enrich-layer/index.ts b/backend/src/services/enrichment/enrich-layer/index.ts deleted file mode 100644 index cc5b8ad25..000000000 --- a/backend/src/services/enrichment/enrich-layer/index.ts +++ /dev/null @@ -1,120 +0,0 @@ -import { Logger } from 'winston'; -import { Engine, EngineResponse, Person } from '../Engine'; -import EnrichLayerAPI, { - ProfileExtra, - ReverseEmailLookupResponse -} from './client'; -import { - undefinedIfEmpty, - undefinedIfFalsy -} from '../../../utils/helpers/validation'; - -export default class EnrichLayer implements Engine { - constructor( - private readonly client: EnrichLayerAPI, - private readonly logger: Logger - ) {} - - readonly name = 'enrichLayer'; - - readonly isSync = true; - - readonly isAsync = false; - - static getProfileUrls(profile: ProfileExtra): string[] { - const urls: string[] = []; - - if (profile?.github_profile_id) { - urls.push(`https://github.com/${profile.github_profile_id}`); - } - if (profile?.facebook_profile_id) { - urls.push(`https://facebook.com/${profile.facebook_profile_id}`); - } - if (profile?.twitter_profile_id) { - urls.push(`https://twitter.com/${profile.twitter_profile_id}`); - } - if (profile?.website) { - urls.push(profile?.website); - } - - return urls; - } - - // eslint-disable-next-line class-methods-use-this - isValid(contact: Partial) { - return Boolean(contact.email); - } - - async enrichSync(person: Partial) { - try { - this.logger.debug(`${this.constructor.name}.enrichSync request`, person); - const response = await this.client.reverseEmailLookup({ - lookup_depth: 'superficial', - enrich_profile: 'enrich', - email: person.email as string - }); - return this.parseResult([response]); - } catch (err) { - throw new Error((err as Error).message); - } - } - - enrichAsync(_: Partial[], __: string): Promise { - this.logger.debug(`${this.constructor.name}.enrichSync request`, _, __); - throw new Error('Method not implemented.'); - } - - parseResult(data: ReverseEmailLookupResponse[]) { - const [response] = data; - const mapped = [ - { - name: undefinedIfFalsy(response?.profile?.full_name ?? ''), - givenName: undefinedIfFalsy(response?.profile?.first_name ?? ''), - familyName: undefinedIfFalsy(response?.profile?.last_name ?? ''), - jobTitle: undefinedIfFalsy(response?.profile?.occupation ?? ''), - organization: undefinedIfFalsy( - response?.profile?.experiences?.[0]?.company ?? '' - ), - image: undefinedIfFalsy(response?.profile?.profile_pic_url ?? ''), - identifiers: undefinedIfEmpty( - [response?.profile?.public_identifier].filter((id): id is string => - Boolean(id) - ) - ), - location: undefinedIfFalsy( - [ - response?.profile?.city, - response?.profile?.state, - response?.profile?.country_full_name - ] - .filter((loc): loc is string => Boolean(loc)) - .join(', ') - ), - sameAs: undefinedIfEmpty([ - response?.linkedin_profile_url, - response?.facebook_profile_url, - response?.twitter_profile_url, - ...EnrichLayer.getProfileUrls(response?.profile?.extra) - ]) - } - ] - .filter( - (result) => - !Array.from(Object.values(result)).every( - (field) => field === undefined || field.length === 0 - ) - ) - .pop(); - - this.logger.debug( - `[${this.constructor.name}]-[parseResult]: Parsing results`, - mapped - ); - - return { - engine: this.name, - data: mapped ? [{ email: response.email, ...mapped }] : [], - raw_data: [response] - }; - } -} diff --git a/backend/src/services/enrichment/index.ts b/backend/src/services/enrichment/index.ts deleted file mode 100644 index bc79d3e7f..000000000 --- a/backend/src/services/enrichment/index.ts +++ /dev/null @@ -1,89 +0,0 @@ -import ENV from '../../config'; -import { Engine } from './Engine'; -import Enricher from './Enricher'; -import Thedig from './thedig'; -import ThedigApi from './thedig/client'; -import Voilanorbert from './voilanorbert'; -import VoilanorbertApi from './voilanorbert/client'; -import logger from '../../utils/logger'; -import { Distribution, TokenBucketRateLimiter } from '../rate-limiter'; -import EnrichLayerAPI from './enrich-layer/client'; -import EnrichLayer from './enrich-layer'; - -let ENGINE_THEDIG: Engine | undefined; -let ENGINE_VOILANORBERT: Engine | undefined; -let ENGINE_ENRICH_LAYER: Engine | undefined; - -if ( - ENV.VOILANORBERT_API_KEY && - ENV.VOILANORBERT_URL && - ENV.VOILANORBERT_USERNAME -) { - ENGINE_VOILANORBERT = new Voilanorbert( - new VoilanorbertApi( - { - url: ENV.VOILANORBERT_URL, - username: ENV.VOILANORBERT_USERNAME, - apiToken: ENV.VOILANORBERT_API_KEY, - rateLimiter: new TokenBucketRateLimiter({ - executeEvenly: true, - uniqueKey: 'email-enrichment-voilanorbert', - distribution: Distribution.Memory, - requests: 115, - intervalSeconds: 60 - }) - }, - logger - ), - logger - ); -} - -if (ENV.THEDIG_API_KEY && ENV.THEDIG_URL) { - ENGINE_THEDIG = new Thedig( - new ThedigApi( - { - url: ENV.THEDIG_URL, - apiToken: ENV.THEDIG_API_KEY, - rateLimiter: new TokenBucketRateLimiter({ - executeEvenly: true, - uniqueKey: 'email-enrichment-theDig', - distribution: Distribution.Memory, - requests: 55, - intervalSeconds: 60 - }) - }, - logger - ), - logger - ); -} - -if (ENV.ENRICH_LAYER_API_KEY && ENV.ENRICH_LAYER_URL) { - ENGINE_ENRICH_LAYER = new EnrichLayer( - new EnrichLayerAPI( - { - url: ENV.ENRICH_LAYER_URL, - apiKey: ENV.ENRICH_LAYER_API_KEY, - rateLimiter: new TokenBucketRateLimiter({ - executeEvenly: true, - uniqueKey: 'email-enrichment-enrichLayer', - distribution: Distribution.Memory, - requests: 295, - intervalSeconds: 60 - }) - }, - logger - ), - logger - ); -} - -const EnrichmentService = new Enricher( - [ENGINE_THEDIG, ENGINE_ENRICH_LAYER, ENGINE_VOILANORBERT].filter( - (engine): engine is Engine => Boolean(engine) - ), - logger -); - -export default EnrichmentService; diff --git a/backend/src/services/enrichment/thedig/client.ts b/backend/src/services/enrichment/thedig/client.ts deleted file mode 100644 index 562a8260b..000000000 --- a/backend/src/services/enrichment/thedig/client.ts +++ /dev/null @@ -1,104 +0,0 @@ -import axios, { AxiosInstance } from 'axios'; - -import { Logger } from 'winston'; -import { logError } from '../../../utils/axios'; -import { IRateLimiter } from '../../rate-limiter'; - -interface Config { - url: string; - apiToken: string; - rateLimiter: IRateLimiter; -} - -export interface EnrichPersonRequest { - url?: string; - name?: string; - email: string; - OptOut?: boolean; - familyName?: string; - givenName?: string; - image?: string[]; - sameAs?: string[]; - jobTitle?: string[]; - worksFor?: string[]; - identifier?: string[]; - nationality?: string[]; - description?: string[]; - homeLocation?: string; - workLocation?: string; - alternateName?: string[]; - knowsLanguage?: string[]; -} - -export interface EnrichPersonResponse { - email: string; - name: string; - givenName: string; - familyName?: string; - alternateName?: string[]; - image?: string[]; - jobTitle?: string[]; - worksFor?: string[]; - homeLocation?: string; - workLocation?: string; - sameAs?: string[]; - identifier?: string[]; - description?: string[]; - knowsLanguage?: string[]; - nationality?: string[]; - OptOut?: boolean; - url?: string; - error_msg?: string; - statusCode: number; -} - -export default class ThedigApi { - private readonly api: AxiosInstance; - - private readonly rateLimiter; - - constructor( - { url, apiToken, rateLimiter }: Config, - private readonly logger: Logger - ) { - this.api = axios.create({ - baseURL: url, - headers: { - 'X-API-KEY': apiToken - } - }); - this.rateLimiter = rateLimiter; - } - - async enrich(person: EnrichPersonRequest) { - try { - const response = await this.rateLimiter.throttleRequests(() => - this.api.post('/person/', person) - ); - return { - ...response.data, - statusCode: response.data.statusCode ?? response.status - }; - } catch (error) { - logError(error, `[${this.constructor.name}:enrich]`, this.logger); - throw error; - } - } - - async enrichBulk(persons: EnrichPersonRequest[], webhook: string) { - try { - const { data } = await this.rateLimiter.throttleRequests(() => - this.api.post(`/person/bulk?endpoint=${webhook}`, persons) - ); - - return { - status: 'running', - success: true, - token: data as string - }; - } catch (error) { - logError(error, `[${this.constructor.name}:enrichBulk]`, this.logger); - throw error; - } - } -} diff --git a/backend/src/services/enrichment/thedig/index.ts b/backend/src/services/enrichment/thedig/index.ts deleted file mode 100644 index 9c7985a5e..000000000 --- a/backend/src/services/enrichment/thedig/index.ts +++ /dev/null @@ -1,149 +0,0 @@ -import { Logger } from 'winston'; -import { - undefinedIfEmpty, - undefinedIfFalsy -} from '../../../utils/helpers/validation'; -import { Engine, EngineResult, Person } from '../Engine'; -import VoilanorbertApi, { - EnrichPersonRequest, - EnrichPersonResponse -} from './client'; - -export default class Thedig implements Engine { - readonly name = 'thedig'; - - readonly isSync = true; - - readonly isAsync = false; - - constructor( - private readonly client: VoilanorbertApi, - private readonly logger: Logger - ) {} - - // eslint-disable-next-line class-methods-use-this - isValid(contact: Partial) { - return Boolean(contact.email && contact.name); - } - - async enrichSync(person: Partial) { - this.logger.debug( - `Got ${this.constructor.name}.enrichSync request`, - person - ); - try { - const personMapped = Thedig.mapForClientRequest(person); - const response = await this.client.enrich(personMapped); - let enrichResponse = this.parseResult([response]); - if (response.statusCode === 203 && !response.sameAs?.length) { - enrichResponse = { engine: this.name, data: [], raw_data: [response] }; - } - return enrichResponse; - } catch (err) { - throw new Error((err as Error).message); - } - } - - async enrichAsync(persons: Partial[], webhook: string) { - this.logger.debug( - `Got ${this.constructor.name}.enrichAsync request`, - persons - ); - try { - const mapped = persons.map(Thedig.mapForClientRequest); - const response = await this.client.enrichBulk( - mapped.map(({ name, email }) => ({ name, email })), - webhook - ); - - if (!response.success) { - throw new Error('Failed to upload emails to enrichment.'); - } - return { - engine: this.name, - token: response.token, - data: [], - raw_data: [] - }; - } catch (err) { - throw new Error((err as Error).message); - } - } - - static mapForClientRequest(person: Partial) { - let personMapped: EnrichPersonRequest = { - email: person.email as string, - name: person.name as string, - homeLocation: person.location, - alternateName: person.alternate_name, - familyName: person.family_name, - givenName: person.given_name, - identifier: person.identifiers, - image: person.image ? [person.image] : undefined, - jobTitle: person.job_title ? [person.job_title] : undefined, - sameAs: person.same_as, - url: person.url, - workLocation: person.location, - worksFor: person.works_for ? [person.works_for] : undefined - }; - - personMapped = Object.fromEntries( - Object.entries(personMapped).filter(([, value]) => Boolean(value)) - ) as EnrichPersonRequest; - return personMapped; - } - - parseResult(enrichedData: EnrichPersonResponse[]) { - this.logger.debug( - `[${this.constructor.name}]-[parseResult]: Parsing enrichment results`, - enrichedData - ); - const results = enrichedData; - const enriched: EngineResult[] = results - .map((person) => ({ - email: person.email, - name: undefinedIfFalsy(person.name), - givenName: undefinedIfFalsy(person.givenName), - familyName: undefinedIfFalsy(person.familyName), - image: undefinedIfFalsy(person.image?.[0]), - jobTitle: undefinedIfFalsy(person.jobTitle?.[0]), - organization: undefinedIfFalsy(person.worksFor?.[0]), - sameAs: undefinedIfEmpty(person.sameAs ?? []), - identifiers: undefinedIfEmpty(person.identifier ?? []), - alternateName: undefinedIfEmpty(person.alternateName ?? []), - location: undefinedIfFalsy( - [person.homeLocation, person.workLocation] - .flat() - .filter(Boolean) - .join(', ') - ) - })) - .filter( - ({ - givenName, - familyName, - sameAs, - organization, - jobTitle, - location, - alternateName, - image - }) => - ![ - givenName, - familyName, - sameAs, - organization, - jobTitle, - location, - alternateName, - image - ].every((field) => !field || field.length === 0) - ); - return { - engine: this.name, - data: enriched, - raw_data: results - }; - } -} diff --git a/backend/src/services/enrichment/voilanorbert/client.ts b/backend/src/services/enrichment/voilanorbert/client.ts deleted file mode 100644 index ef0b49530..000000000 --- a/backend/src/services/enrichment/voilanorbert/client.ts +++ /dev/null @@ -1,78 +0,0 @@ -import axios, { AxiosInstance } from 'axios'; - -import { Logger } from 'winston'; -import qs from 'qs'; -import { logError } from '../../../utils/axios'; -import { IRateLimiter } from '../../rate-limiter'; - -interface Config { - url: string; - username: string; - apiToken: string; - rateLimiter: IRateLimiter; -} - -interface Result { - email: string; - fullName: string; - title: string; - organization: string; - location: string; - twitter: string; - linkedin: string; - facebook: string; - error_msg?: string; -} - -export interface ResponseAsync { - status: string; - success: boolean; - token: string; -} - -export interface ResponseWebhook { - id: string; - token: string; - results: Result[]; -} - -export default class VoilanorbertApi { - private static readonly baseURL = 'https://api.voilanorbert.com/2018-01-08/'; - - private readonly api: AxiosInstance; - - private readonly rateLimiter; - - constructor( - { url, username, apiToken, rateLimiter }: Config, - private readonly logger: Logger - ) { - this.api = axios.create({ - baseURL: url ?? VoilanorbertApi.baseURL, - headers: {}, - auth: { - username, - password: apiToken - } - }); - this.rateLimiter = rateLimiter; - } - - async enrich(emails: string[], webhook: string): Promise { - try { - const { data } = await this.rateLimiter.throttleRequests(() => - this.api.post( - '/enrich/upload', - qs.stringify({ - data: emails.join('\n'), - webhook - }) - ) - ); - return data; - } catch (error) { - logError(error, `[${this.constructor.name}:enrich]`, this.logger); - throw error; - } - } -} diff --git a/backend/src/services/enrichment/voilanorbert/index.ts b/backend/src/services/enrichment/voilanorbert/index.ts deleted file mode 100644 index 89ba53657..000000000 --- a/backend/src/services/enrichment/voilanorbert/index.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { Logger } from 'winston'; -import { Engine, EngineResponse, Person } from '../Engine'; -import VoilanorbertApi, { ResponseWebhook } from './client'; -import { - undefinedIfEmpty, - undefinedIfFalsy -} from '../../../utils/helpers/validation'; - -export default class Voilanorbert implements Engine { - readonly name = 'voilanorbert'; - - readonly isSync = false; - - readonly isAsync = true; - - constructor( - private readonly client: VoilanorbertApi, - private readonly logger: Logger - ) {} - - // eslint-disable-next-line class-methods-use-this - isValid(contact: Partial) { - return Boolean(contact.email); - } - - enrichSync(person: Partial): Promise { - this.logger.debug( - `Got ${this.constructor.name}.enrichSync request`, - person - ); - throw new Error( - `[${this.constructor.name}]: method enrichSync not implemented.` - ); - } - - async enrichAsync(persons: Partial[], webhook: string) { - this.logger.debug( - `Got ${this.constructor.name}.enrichAsync request`, - persons - ); - try { - const response = await this.client.enrich( - persons.map(({ email }) => email as string), - webhook - ); - - if (!response.success) { - throw new Error('Failed to upload emails to enrichment.'); - } - - return { - engine: this.name, - token: response.token, - data: [], - raw_data: [] - }; - } catch (err) { - throw new Error((err as Error).message); - } - } - - parseResult(enrichedData: unknown[]) { - this.logger.debug( - `[${this.constructor.name}]-[parseResult]: Parsing enrichment results`, - enrichedData - ); - const results = - (enrichedData[0] as ResponseWebhook).results ?? - (enrichedData as ResponseWebhook['results']); - const enriched = results - .map((result) => ({ - email: result.email, - image: undefinedIfFalsy(''), - name: undefinedIfFalsy(result.fullName), - organization: undefinedIfFalsy(result.organization), - jobTitle: undefinedIfFalsy(result.title), - location: undefinedIfFalsy(result.location), - sameAs: undefinedIfEmpty([ - result.facebook, - result.linkedin, - result.twitter - ]) - })) - .filter( - ({ email, name, location, organization, jobTitle, sameAs }) => - email !== 'Email' && - ![name, location, organization, jobTitle, sameAs].every( - (field) => field === undefined || field.length === 0 - ) - ); - return { - engine: this.name, - data: enriched, - raw_data: results - }; - } -} diff --git a/backend/src/services/export/exports/csv.ts b/backend/src/services/export/exports/csv.ts deleted file mode 100644 index 03dd3fdc3..000000000 --- a/backend/src/services/export/exports/csv.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { stringify } from 'csv-stringify'; -import { Contact } from '../../../db/types'; -import { ExportStrategy, ExportType, ExportResult } from '../types'; - -export default class CsvExport implements ExportStrategy { - readonly contentType = 'text/csv'; - - readonly type = ExportType.CSV; - - async export( - contacts: Contact[], - options?: { locale: string; delimiter: string } - ): Promise { - const delimiter = - options?.delimiter ?? CsvExport.getLocalizedCsvSeparator(options?.locale); - - const csvData = contacts.map((contact) => ({ - name: contact.name?.trim(), - email: contact.email, - recency: contact.recency - ? new Date(contact.recency).toISOString().slice(0, 10) - : '', - seniority: contact.seniority - ? new Date(contact.seniority).toISOString().slice(0, 10) - : '', - occurrence: contact.occurrence, - sender: contact.sender, - recipient: contact.recipient, - conversations: contact.conversations, - replied_conversations: contact.replied_conversations, - tags: contact.tags?.join(','), - status: contact.status, - given_name: contact.given_name, - family_name: contact.family_name, - alternate_name: contact.alternate_name?.join(','), - location: contact.location, - works_for: contact.works_for, - job_title: contact.job_title, - same_as: contact.same_as?.join(','), - telephone: contact.telephone?.join(','), - image: contact.image - })); - - const content = await CsvExport.getCsvStr( - [ - { key: 'name', header: 'Name' }, - { key: 'email', header: 'Email' }, - { key: 'recency', header: 'Recency' }, - { key: 'seniority', header: 'Seniority' }, - { key: 'occurrence', header: 'Occurrence' }, - { key: 'sender', header: 'Sender' }, - { key: 'recipient', header: 'Recipient' }, - { key: 'conversations', header: 'Conversations' }, - { key: 'replied_conversations', header: 'Replies' }, - { key: 'tags', header: 'Tags' }, - { key: 'status', header: 'Reachable' }, - { key: 'given_name', header: 'Given name' }, - { key: 'family_name', header: 'Family name' }, - { key: 'alternate_name', header: 'Alternate names' }, - { key: 'location', header: 'Location' }, - { key: 'works_for', header: 'Works for' }, - { key: 'job_title', header: 'Job title' }, - { key: 'same_as', header: 'Same as' }, - { key: 'telephone', header: 'Telephone' }, - { key: 'image', header: 'Avatar URL' } - ], - csvData, - delimiter - ); - - return { - content, - contentType: this.contentType, - charset: 'utf-8', - extension: 'csv' - }; - } - - static getLocalizedCsvSeparator(locale?: string) { - const language = locale?.substring(0, 2); - - switch (language) { - case 'fr': - case 'de': - case 'es': - case 'pt': - case 'it': - return ';'; - default: - return ','; - } - } - - static getCsvStr( - columns: { key: keyof T; header: string }[], - rows: T[], - delimiter: string - ) { - return new Promise((resolve, reject) => { - stringify( - rows, - { - columns: columns.map(({ key, header }) => ({ - key: String(key), - header - })), - bom: true, - delimiter, - header: true, - quoted_string: true - }, - (err, data) => { - if (err) { - return reject(err); - } - return resolve(data); - } - ); - }); - } -} diff --git a/backend/src/services/export/exports/google/contacts-api.ts b/backend/src/services/export/exports/google/contacts-api.ts deleted file mode 100644 index 4c46c8ed8..000000000 --- a/backend/src/services/export/exports/google/contacts-api.ts +++ /dev/null @@ -1,533 +0,0 @@ -import { people_v1 } from 'googleapis'; -import { ContactFrontend } from '../../../../db/types'; -import logger from '../../../../utils/logger'; -import { Distribution, TokenBucketRateLimiter } from '../../../rate-limiter'; -import ENV from '../../../../config'; - -export type QuotaType = 'criticalRead' | 'criticalWrite' | 'read' | 'write'; - -export default class GoogleContactsSession { - private readonly PERSON_FIELDS = - 'names,emailAddresses,phoneNumbers,organizations,addresses,nicknames,urls,memberships'; - - private readonly FULL_READ_MASK = `${this.PERSON_FIELDS},metadata`; - - private labelMap = new Map(); - - private service: people_v1.People; - - private limiters: Record; - - constructor( - service: people_v1.People, - private readonly appName: string, - private readonly userId: string - ) { - this.service = service; - - const config = { - criticalRead: { - requests: ENV.GOOGLE_CONTACTS_CRITICAL_READ_REQUESTS, - intervalSeconds: ENV.GOOGLE_CONTACTS_CRITICAL_READ_INTERVAL, - uniqueKey: `crit-read-${this.userId}`, - executeEvenly: true, - distribution: Distribution.Memory - }, // Limit is 90 - criticalWrite: { - requests: ENV.GOOGLE_CONTACTS_CRITICAL_WRITE_REQUESTS, - intervalSeconds: ENV.GOOGLE_CONTACTS_CRITICAL_WRITE_INTERVAL, - uniqueKey: `crit-write-${this.userId}`, - executeEvenly: true, - distribution: Distribution.Memory - }, // Limit is 90 - read: { - requests: ENV.GOOGLE_CONTACTS_READ_REQUESTS, - intervalSeconds: ENV.GOOGLE_CONTACTS_READ_INTERVAL, - uniqueKey: `read-${this.userId}`, - executeEvenly: true, - distribution: Distribution.Memory - }, // Limit is 120 (Groups) - write: { - requests: ENV.GOOGLE_CONTACTS_WRITE_REQUESTS, - intervalSeconds: ENV.GOOGLE_CONTACTS_WRITE_INTERVAL, - uniqueKey: `write-${this.userId}`, - executeEvenly: true, - distribution: Distribution.Memory - } // Limit is 90 (Groups/Delete) - }; - - this.limiters = { - criticalRead: new TokenBucketRateLimiter(config.criticalRead), - criticalWrite: new TokenBucketRateLimiter(config.criticalWrite), - read: new TokenBucketRateLimiter(config.read), - write: new TokenBucketRateLimiter(config.write) - }; - } - - private async useQuota( - requirements: { type: QuotaType; weight: number }[], - callback: () => Promise - ): Promise { - await Promise.all( - requirements.map((q) => this.limiters[q.type].removeTokens(q.weight)) - ); - return callback(); - } - - private static batchPerLimit(array: T[], size: number): T[][] { - const chunks: T[][] = []; - for (let i = 0; i < array.length; i += size) { - chunks.push(array.slice(i, i + size)); - } - return chunks; - } - - async run( - contacts: ContactFrontend[], - updateEmptyOnly: boolean - ): Promise<{ labelId: string | null }> { - try { - logger.debug('GoogleContactsSession.run(): Starting sync process', { - userId: this.userId, - contactCount: contacts.length, - updateEmptyOnly - }); - - const existingLabels = await this.listLabels(); - - logger.debug('GoogleContactsSession.run(): Fetched labels fetched', { - labelCount: existingLabels.size, - labels: Array.from(existingLabels.keys()) - }); - - const contactsLabels = [{ tags: [ENV.APP_NAME] }, ...contacts] - .map(({ tags }) => tags) - .flat() - .filter((tag): tag is string => Boolean(tag)); - - logger.debug('GoogleContactsSession.run(): Extracted contact tags', { - uniqueTagCount: new Set(contactsLabels).size, - tags: [...new Set(contactsLabels)] - }); - - const created = await this.createLabels(existingLabels, contactsLabels); - - this.labelMap = new Map([...existingLabels, ...created]); - - logger.debug( - 'GoogleContactsSession.run(): Created missing labels and build labelMap', - { - newLabelCount: created.size, - newLabels: Array.from(created.keys()), - totalLabels: this.labelMap.size - } - ); - - const { create, update } = await this.contactsToCreateUpdate(contacts); - - const createBatches = GoogleContactsSession.batchPerLimit( - Array.from(create.values()), - 200 - ); - const updateBatches = GoogleContactsSession.batchPerLimit( - Array.from(update.values()), - 200 - ); - - logger.debug('GoogleContactsSession.run(): Batches prepared', { - createBatches: createBatches.length, - updateBatches: updateBatches.length - }); - - /* eslint-disable no-await-in-loop */ - for (let i = 0; i < createBatches.length; i += 1) { - const batch = createBatches[i]; - logger.debug( - `GoogleContactsSession.run(): Processing create batch ${i + 1}/${createBatches.length}`, - { - batchSize: batch.length - } - ); - - await this.useQuota( - [ - { type: 'criticalRead', weight: 6 }, - { type: 'criticalWrite', weight: 6 } - ], - async () => { - await this.executeBatchCreate(batch); - } - ); - - logger.debug( - `GoogleContactsSession.run(): Create batch ${i + 1}/${createBatches.length} completed` - ); - } - - /* eslint-disable no-await-in-loop */ - for (let i = 0; i < updateBatches.length; i += 1) { - const batch = updateBatches[i]; - logger.debug( - `GoogleContactsSession.run(): Processing update batch ${i + 1}/${updateBatches.length}`, - { - batchSize: batch.length - } - ); - - await this.useQuota( - [ - { type: 'criticalRead', weight: 6 }, - { type: 'criticalWrite', weight: 6 } - ], - async () => { - await this.executeBatchUpdate(batch, updateEmptyOnly); - } - ); - - logger.debug( - `GoogleContactsSession.run(): Update batch ${i + 1}/${updateBatches.length} completed` - ); - } - - logger.info('GoogleContactsSession.run(): Sync completed successfully', { - totalContacts: contacts.length, - created: create.size, - updated: update.size - }); - - const appLabelResourceName = this.labelMap.get( - ENV.APP_NAME.toLowerCase() - ); - const labelId = appLabelResourceName - ? appLabelResourceName.replace('contactGroups/', '') - : null; - - return { labelId }; - } catch (err) { - logger.error( - `GoogleContactsSession.run(): Error during sync: ${(err as Error).message}`, - err - ); - throw err; - } - } - - private async contactsToCreateUpdate(contacts: ContactFrontend[]) { - const create: Map = new Map(); - const update: Map< - string, - { existing: people_v1.Schema$Person; incoming: ContactFrontend } - > = new Map(); - - const { emailMap, phoneMap } = await this.getUserContacts(); - - for (const contact of contacts) { - const potentialMatches = new Map(); - - const emailMatches = contact.email - ? emailMap.get(contact.email.toLowerCase()) || [] - : []; - emailMatches.forEach((p) => { - if (p.resourceName) potentialMatches.set(p.resourceName, p); - }); - - if (!emailMatches.length && contact.telephone) { - for (const phone of contact.telephone) { - const normalizedPhone = phone.replace(/\s+/g, ''); - const phoneMatches = phoneMap.get(normalizedPhone) || []; - phoneMatches.forEach((p) => { - if (p.resourceName) potentialMatches.set(p.resourceName, p); - }); - } - } - - if (potentialMatches.size === 1) { - const existing = Array.from(potentialMatches.values())[0]; - update.set(existing.resourceName as string, { - existing, - incoming: contact - }); - } else { - // Use identifier (email or phone) as the dedup key in `create`. - // Phone-only contacts have no email, so the map key is the - // primary phone number. - const key = contact.email ?? contact.telephone?.[0] ?? contact.id; - if (key) { - create.set(key, contact); - } - if (potentialMatches.size > 1) { - logger.warn( - `Ambiguous match for ${contact.email ?? contact.telephone?.[0]}: Found ${potentialMatches.size} contacts. Creating new record to avoid corruption.` - ); - } - } - } - return { create, update }; - } - - private async listLabels(): Promise> { - const labels = new Map(); - await this.useQuota([{ type: 'read', weight: 1 }], async () => { - const res = await this.service.contactGroups.list({ - groupFields: 'name' - }); - (res.data.contactGroups || []).forEach((g) => { - if (g.name && g.resourceName) - labels.set(g.name.toLowerCase(), g.resourceName); - }); - }); - return labels; - } - - private async createLabels( - existing: Map, - labels: string[] - ): Promise> { - const labelsMap = new Map(); - const toCreate = [...new Set(labels)].filter( - (label) => !existing.has(label?.toLowerCase()) - ); - const tasks = toCreate.map(async (tag) => { - await this.useQuota( - [ - { type: 'write', weight: 1 }, - { type: 'read', weight: 1 } - ], - async () => { - const newGroup = await this.service.contactGroups.create({ - requestBody: { contactGroup: { name: tag } } - }); - - if (newGroup.data.resourceName) - labelsMap.set(tag.toLowerCase(), newGroup.data.resourceName); - } - ); - }); - - await Promise.all(tasks); - - return labelsMap; - } - - private async executeBatchUpdate( - batch: { existing: people_v1.Schema$Person; incoming: ContactFrontend }[], - updateEmptyOnly: boolean - ) { - const contactsMap: { [key: string]: people_v1.Schema$Person } = {}; - - batch.forEach((item) => { - const person = this.mapToPerson( - item.incoming, - item.existing, - updateEmptyOnly - ); - if (item.existing.resourceName) { - contactsMap[item.existing.resourceName] = { - ...person, - etag: item.existing.etag - }; - } - }); - - if (Object.keys(contactsMap).length === 0) return; - - await this.service.people.batchUpdateContacts({ - requestBody: { - contacts: contactsMap, - updateMask: this.PERSON_FIELDS - } - }); - logger.info(`Batch updated ${batch.length} contacts.`); - } - - private async executeBatchCreate(batch: ContactFrontend[]) { - const res = await this.service.people.batchCreateContacts({ - requestBody: { - contacts: batch.map((c) => ({ contactPerson: this.mapToPerson(c) })) - } - }); - logger.info(`Batch created ${batch.length} contacts.`); - return res.data; - } - - private async getUserContacts(): Promise<{ - emailMap: Map; - phoneMap: Map; - }> { - const emailMap = new Map(); - const phoneMap = new Map(); - let pageToken: string | undefined; - - do { - const currentPageToken = pageToken; - const res = await this.useQuota( - [{ type: 'criticalRead', weight: 1 }], - async () => - this.service.people.connections.list({ - resourceName: 'people/me', - pageSize: 1000, - pageToken: currentPageToken, - personFields: this.FULL_READ_MASK - }) - ); - - const connections = (res.data.connections || []).filter((p) => - p.metadata?.sources?.some((s) => s.type === 'CONTACT') - ); - - for (const person of connections) { - person.emailAddresses?.forEach((e) => { - if (!e.value) return; - const key = e.value.toLowerCase(); - const existing = emailMap.get(key) || []; - emailMap.set(key, [...existing, person]); - }); - - person.phoneNumbers?.forEach((p) => { - if (!p.value) return; - const key = p.value.replace(/\s+/g, ''); - const existing = phoneMap.get(key) || []; - phoneMap.set(key, [...existing, person]); - }); - } - - pageToken = res.data.nextPageToken ?? undefined; - } while (pageToken); - - return { emailMap, phoneMap }; - } - - // skipcq: JS-R1005 - Pre-existing on main; refactor tracked in #2831. Function maps 7+ schema field types (names, emails, phones, orgs, urls, addresses, memberships) in one pass and the per-field "is duplicate" checks against existing fields inherently inflate cyclomatic complexity. Splitting into helpers would be a behavioral change and out of scope for PR #2830. - private mapToPerson( - contact: ContactFrontend, - existing?: people_v1.Schema$Person, - updateEmptyOnly = false - ): people_v1.Schema$Person { - const existingUrls = existing?.urls || []; - const existingNames = existing?.names || []; - const existingOrgs = existing?.organizations || []; - const existingAddresses = existing?.addresses || []; - const existingPhones = existing?.phoneNumbers || []; - const existingEmails = existing?.emailAddresses || []; - const existingMemberships = existing?.memberships || []; - const labels = [this.appName, ...(contact.tags ?? [])].filter(Boolean); - - // Helper function to check if a field is empty - const isEmpty = (val: string | null | undefined) => - val === null || val === undefined || val === ''; - - let names: people_v1.Schema$Name[] = existingNames; - if (!updateEmptyOnly || existingNames.length === 0) { - const newName: people_v1.Schema$Name = {}; - if (!isEmpty(contact.given_name)) { - newName.givenName = contact.given_name; - } - if (!isEmpty(contact.family_name)) { - newName.familyName = contact.family_name; - } - if (!isEmpty(contact.name)) { - newName.unstructuredName = contact.name; - } - - if (newName.givenName || newName.familyName || newName.unstructuredName) { - names = [newName]; - } - } - - const hasEmail = !isEmpty(contact.email); - const emailAddresses = - !updateEmptyOnly || existingEmails.length === 0 - ? [ - ...existingEmails.filter( - (e) => hasEmail && e.value !== contact.email - ), - ...(hasEmail ? [{ value: contact.email as string }] : []) - ] - : existingEmails; - - let phoneNumbers: people_v1.Schema$PhoneNumber[] = existingPhones; - const newPhones = - contact.telephone - ?.filter((tel) => !isEmpty(tel)) - .map((tel) => ({ value: tel })) || []; - if (newPhones.length > 0) { - const existingValues = new Set(existingPhones.map((p) => p.value)); - const uniqueNewPhones = newPhones.filter( - (p) => !existingValues.has(p.value) - ); - if (uniqueNewPhones.length > 0) { - phoneNumbers = [...existingPhones, ...uniqueNewPhones]; - } - } - - let organizations: people_v1.Schema$Organization[] = existingOrgs; - const newOrg: people_v1.Schema$Organization = {}; - if (!isEmpty(contact.works_for)) { - newOrg.name = contact.works_for; - } - if (!isEmpty(contact.job_title)) { - newOrg.title = contact.job_title; - } - if (newOrg.name || newOrg.title) { - const isDuplicate = existingOrgs.some( - (o) => o.name === newOrg.name && o.title === newOrg.title - ); - if (!isDuplicate) { - organizations = [...existingOrgs, newOrg]; - } - } - - let urls: people_v1.Schema$Url[] = existingUrls; - const newUrls = - contact.same_as - ?.filter((url) => !isEmpty(url)) - .map((url) => ({ value: url })) || []; - if (newUrls.length > 0) { - const existingValues = new Set(existingUrls.map((u) => u.value)); - const uniqueNewUrls = newUrls.filter((u) => !existingValues.has(u.value)); - if (uniqueNewUrls.length > 0) { - urls = [...existingUrls, ...uniqueNewUrls]; - } - } - - let addresses: people_v1.Schema$Address[] = existingAddresses; - if (!isEmpty(contact.location)) { - const newAddress = { streetAddress: contact.location }; - const isDuplicate = existingAddresses.some( - (a) => a.streetAddress === newAddress.streetAddress - ); - if (!isDuplicate) { - addresses = [...existingAddresses, newAddress]; - } - } - - const person: people_v1.Schema$Person = { - names, - emailAddresses, - phoneNumbers, - organizations, - urls, - addresses, - memberships: [ - ...existingMemberships.filter( - (m) => - !labels.some( - (l) => - this.labelMap.get(l.toLowerCase()) === - m.contactGroupMembership?.contactGroupResourceName - ) - ), - ...labels - .map((l) => this.labelMap.get(l.toLowerCase())) - .filter(Boolean) - .map((groupResourceName) => ({ - contactGroupMembership: { - contactGroupResourceName: groupResourceName - } - })) - ] - }; - - return person; - } -} diff --git a/backend/src/services/export/exports/google/index.ts b/backend/src/services/export/exports/google/index.ts deleted file mode 100644 index 958abe5a3..000000000 --- a/backend/src/services/export/exports/google/index.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { GaxiosError } from 'gaxios'; -import ENV from '../../../../config'; -import { ContactFrontend } from '../../../../db/types'; -import logger from '../../../../utils/logger'; -import { - getPeopleService, - warmupSearchIndex -} from '../../../OAuth2/googlePeopleClient'; -import { - ExportStrategy, - ExportType, - ExportOptions, - ExportResult -} from '../../types'; -import GoogleContactsSession from './contacts-api'; - -export default class GoogleContactsExport - implements ExportStrategy -{ - readonly type = ExportType.GOOGLE_CONTACTS; - - readonly contentType = 'application/json'; - - async export( - contacts: ContactFrontend[], - options: ExportOptions - ): Promise { - if (!options?.googleContactsOptions) - throw new Error('Missing required options'); - - try { - const opts = options.googleContactsOptions; - const service = await getPeopleService({ - accessToken: opts.accessToken, - refreshToken: opts.refreshToken - }); - await warmupSearchIndex(service); - const session = new GoogleContactsSession( - service, - ENV.APP_NAME, - opts.userId - ); - const { labelId } = await session.run( - contacts, - opts.updateEmptyFieldsOnly ?? false - ); - - return { - content: JSON.stringify({ labelId }), - contentType: this.contentType, - charset: '', - extension: '' - }; - } catch (err) { - if (err instanceof GaxiosError) { - logger.error(`Export error: ${err.message}`, err); - if ( - err.response?.status === 401 || - err.response?.data.error === 'invalid_grant' - ) { - throw new Error('Invalid credentials.'); - } - } else { - logger.error(`Export error: ${(err as Error).message}`, err); - } - throw err; - } - } -} diff --git a/backend/src/services/export/exports/vcard.ts b/backend/src/services/export/exports/vcard.ts deleted file mode 100644 index 06821634f..000000000 --- a/backend/src/services/export/exports/vcard.ts +++ /dev/null @@ -1,69 +0,0 @@ -import VCard from 'vcard-creator'; -import { Contact } from '../../../db/types'; -import { ExportStrategy, ExportType, ExportResult } from '../types'; -import ENV from '../../../config'; - -export default class VCardExport implements ExportStrategy { - readonly contentType = 'text/vcard'; - - readonly type = ExportType.VCARD; - - async export(contacts: Contact[]): Promise { - const content = contacts - .map((contact) => VCardExport.contactToVCard(contact)) - .join('\n'); - - return { - content, - contentType: this.contentType, - charset: 'utf-8', - extension: 'vcf' - }; - } - - private static contactToVCard(contact: Contact): string { - const vcard = new VCard(); - - vcard.addCategories([ENV.APP_NAME]); - - if (contact.given_name?.length || contact.family_name?.length) { - vcard.addName(contact.family_name ?? '', contact.given_name ?? ''); - } else if (contact.name) { - vcard.addName(contact.name); - } - - if (contact.email) { - vcard.addEmail(contact.email); - } - - contact.telephone?.forEach((phone) => { - vcard.addPhoneNumber(phone); - }); - - if (contact.works_for) { - vcard.addCompany(contact.works_for); - } - - if (contact.job_title) { - vcard.addJobtitle(contact.job_title); - } - - // Address / location - if (contact.location) { - vcard.addAddress(contact.location); - } - - if (contact.alternate_name) - vcard.addNickname(contact.alternate_name.join(',')); - - contact.same_as?.forEach((url) => { - vcard.addURL(url); - }); - - if (contact.image) { - vcard.addPhotoURL(contact.image); - } - - return vcard.toString(); - } -} diff --git a/backend/src/services/export/types.ts b/backend/src/services/export/types.ts deleted file mode 100644 index 79b6478b2..000000000 --- a/backend/src/services/export/types.ts +++ /dev/null @@ -1,30 +0,0 @@ -export enum ExportType { - CSV = 'csv', - VCARD = 'vcard', - GOOGLE_CONTACTS = 'google_contacts' -} - -export interface ExportOptions { - delimiter?: string; - locale?: string; - googleContactsOptions?: { - userId: string; - accessToken?: string; - refreshToken?: string; - updateEmptyFieldsOnly?: boolean; - }; -} - -export interface ExportResult { - content: string | Buffer; - contentType: string; - charset: string; - extension: string; - metadata?: Record; -} - -export interface ExportStrategy { - readonly type: ExportType; - - export(data: T[], options?: ExportOptions): Promise; -} diff --git a/backend/src/services/imap/ImapBoxesFetcher.ts b/backend/src/services/imap/ImapBoxesFetcher.ts deleted file mode 100644 index 868dbcb22..000000000 --- a/backend/src/services/imap/ImapBoxesFetcher.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { ImapFlow as Connection } from 'imapflow'; -import { Logger } from 'winston'; -import { - buildFinalTree, - createFlatTreeFromImap -} from '../../utils/helpers/imapTreeHelpers'; -import { EXCLUDED_IMAP_FOLDERS } from '../../utils/constants'; - -export default class ImapBoxesFetcher { - constructor( - private readonly imapConnection: Connection, - private readonly logger: Logger - ) {} - - /** - * Retrieves the IMAP tree of the email account. - * @returns IMAP tree. - */ - async getTree(userEmail: string) { - const tree = await this.imapConnection.list({ - statusQuery: { messages: true } - }); - - return buildFinalTree(createFlatTreeFromImap(tree), userEmail); - } - - /** - * Fetches the total number of messages across the specified folders on an IMAP server. - */ - async getTotalMessages(inboxes: string[]) { - let total = 0; - - try { - // Create an array of Promises that resolve to the total number of messages in each folder. - const folders = inboxes.filter( - (folder) => !EXCLUDED_IMAP_FOLDERS.includes(folder) - ); - - folders.forEach(async (folder) => { - try { - const status = await this.imapConnection?.status(folder, { - messages: true - }); - total += status?.messages ?? 0; - } catch (err) { - this.logger.warn(`Could not STATUS ${folder}`, err); - } - }); - return total; - } catch (err) { - this.logger.error('Failed fetching total messages', { - folders: inboxes, - error: err - }); - throw err; - } - } -} diff --git a/backend/test/unit/contact-export/googleContacts.test.ts b/backend/test/unit/contact-export/googleContacts.test.ts deleted file mode 100644 index c15f7d4e6..000000000 --- a/backend/test/unit/contact-export/googleContacts.test.ts +++ /dev/null @@ -1,1150 +0,0 @@ -import { google, people_v1 } from 'googleapis'; -import { GaxiosResponse } from 'gaxios'; -import { jest, describe, beforeEach, it, expect } from '@jest/globals'; -import { AxiosError } from 'axios'; -import { ContactFrontend } from '../../../src/db/types'; -import GoogleContactsSession from '../../../src/services/export/exports/google/contacts-api'; -import GoogleContactsExport from '../../../src/services/export/exports/google'; -import { ExportOptions } from '../../../src/services/export/types'; -import logger from '../../../src/utils/logger'; - -// Mock Config & Logger -jest.mock('../../../src/config', () => ({ - APP_NAME: 'leadminer-test', - GOOGLE_CONTACTS_READ_REQUESTS: 1000, - GOOGLE_CONTACTS_WRITE_REQUESTS: 1000, - GOOGLE_CONTACTS_CRITICAL_READ_REQUESTS: 1000, - GOOGLE_CONTACTS_CRITICAL_WRITE_REQUESTS: 1000, - - GOOGLE_CONTACTS_READ_INTERVAL: 60, - GOOGLE_CONTACTS_WRITE_INTERVAL: 60, - GOOGLE_CONTACTS_CRITICAL_READ_INTERVAL: 60, - GOOGLE_CONTACTS_CRITICAL_WRITE_INTERVAL: 60 -})); -jest.mock('ioredis'); -jest.mock('../../../src/utils/logger'); -jest.mock('../../../src/utils/redis'); - -jest.mock('googleapis', () => ({ - google: { - people: jest.fn(), - auth: { - OAuth2: jest.fn().mockImplementation(() => ({ - setCredentials: jest.fn(), - credentials: {} - })) - } - } -})); - -type SearchContactsFn = ( - params: people_v1.Params$Resource$People$Searchcontacts -) => Promise>; - -type ConnectionsListFn = ( - params: people_v1.Params$Resource$People$Connections$List -) => Promise>; - -type BatchCreateFn = ( - params: people_v1.Params$Resource$People$Batchcreatecontacts -) => Promise>; - -type BatchUpdateFn = ( - params: people_v1.Params$Resource$People$Batchupdatecontacts -) => Promise>; - -type ContactGroupsListFn = ( - params: people_v1.Params$Resource$Contactgroups$List -) => Promise>; - -type ContactGroupsCreateFn = ( - params: people_v1.Params$Resource$Contactgroups$Create -) => Promise>; - -interface MockedPeopleService { - people: { - searchContacts?: jest.MockedFunction; - connections: { - list: jest.MockedFunction; - }; - batchCreateContacts: jest.MockedFunction; - batchUpdateContacts: jest.MockedFunction; - }; - contactGroups: { - list: jest.MockedFunction; - create: jest.MockedFunction; - }; -} - -describe('GoogleContactsExport', () => { - let exporter: GoogleContactsExport; - let mockService: MockedPeopleService; - - const mockOptions: ExportOptions = { - googleContactsOptions: { - userId: 'user-123', - accessToken: 'valid_token', - refreshToken: 'valid_refresh', - updateEmptyFieldsOnly: false - } - }; - - function createMockRes(data: T): GaxiosResponse { - const response = { - data, - status: 200, - statusText: 'OK', - headers: new Headers(), - config: { - url: 'https://mock.api', - method: 'GET' - } - }; - - return response as unknown as GaxiosResponse; - } - - function createMockContact( - overrides: Partial = {} - ): ContactFrontend { - return { - id: '1', - user_id: 'user-123', - email: 'test@example.com', - name: 'Test User', - given_name: 'Test', - family_name: 'User', - ...overrides - } as ContactFrontend; - } - - beforeEach(() => { - jest.clearAllMocks(); - - // Create properly typed mock functions - const searchContactsMock = jest - .fn() - .mockResolvedValue(createMockRes({ results: [] })); - - const connectionsListMock = jest - .fn() - .mockResolvedValue(createMockRes({ connections: [] })); - - const batchCreateMock = jest - .fn() - .mockResolvedValue(createMockRes({})); - - const batchUpdateMock = jest - .fn() - .mockResolvedValue(createMockRes({})); - - const groupsListMock = jest - .fn() - .mockResolvedValue(createMockRes({ contactGroups: [] })); - - const groupsCreateMock = jest - .fn() - .mockResolvedValue(createMockRes({ resourceName: 'groups/new_id' })); - - mockService = { - people: { - searchContacts: searchContactsMock, - connections: { - list: connectionsListMock - }, - batchCreateContacts: batchCreateMock, - batchUpdateContacts: batchUpdateMock - }, - contactGroups: { - list: groupsListMock, - create: groupsCreateMock - } - }; - - const mockGooglePeople = google.people as jest.MockedFunction< - typeof google.people - >; - mockGooglePeople.mockReturnValue( - mockService as unknown as people_v1.People - ); - - exporter = new GoogleContactsExport(); - }); - - it('should initialize service and trigger warmup exactly once', async () => { - await exporter.export([], mockOptions); - - expect(google.people).toHaveBeenCalledWith({ - version: 'v1', - auth: expect.any(Object) - }); - - expect(mockService.people.searchContacts).toHaveBeenCalledWith( - expect.objectContaining({ query: '' }) - ); - - expect(mockService.people.connections.list).toHaveBeenCalledWith( - expect.objectContaining({ - resourceName: 'people/me', - pageSize: 1 - }) - ); - }); - - it('should reject with Error if accessToken is missing', async () => { - const invalidOptions = { - googleContactsOptions: {} - } as ExportOptions; - - await expect(exporter.export([], invalidOptions)).rejects.toThrow( - 'Invalid credentials.' - ); - }); - - it('should throw and log error if the People Service initialization fails (e.g. 401)', async () => { - const authError = new Error('Unauthorized') as AxiosError; - authError.status = 401; - - (google.people as jest.Mock).mockImplementationOnce(() => { - throw authError; - }); - - await expect(exporter.export([], mockOptions)).rejects.toThrow( - 'Unauthorized' - ); - - expect(logger.error).toHaveBeenCalledWith( - expect.stringContaining('Export error: Unauthorized'), - expect.anything() - ); - }); - - it('should bubble up critical errors from the session run', async () => { - const failingBatchCreate = jest - .fn() - .mockRejectedValue(new Error('Quota Exceeded')); - - const failingService: MockedPeopleService = { - people: { - searchContacts: jest - .fn() - .mockResolvedValue(createMockRes({ results: [] })), - connections: { - list: jest - .fn() - .mockResolvedValue(createMockRes({ connections: [] })) - }, - batchCreateContacts: failingBatchCreate, - batchUpdateContacts: jest - .fn() - .mockResolvedValue(createMockRes({})) - }, - contactGroups: { - list: jest - .fn() - .mockResolvedValue(createMockRes({ contactGroups: [] })), - create: jest - .fn() - .mockResolvedValue(createMockRes({ resourceName: 'groups/1' })) - } - }; - - (google.people as jest.Mock).mockReturnValue( - failingService as unknown as people_v1.People - ); - - const instance = new GoogleContactsExport(); - const contact = createMockContact({ email: 'test@test.com' }); - const options: ExportOptions = { - googleContactsOptions: { - userId: 'user-123', - accessToken: 'fake_token', - updateEmptyFieldsOnly: false - } - }; - - await expect(instance.export([contact], options)).rejects.toThrow( - 'Quota Exceeded' - ); - - expect(logger.error).toHaveBeenCalledTimes(2); - expect(logger.error).toHaveBeenNthCalledWith( - 1, - expect.stringContaining('Error during sync:'), - expect.any(Error) - ); - expect(logger.error).toHaveBeenNthCalledWith( - 2, - expect.stringContaining('Export error:'), - expect.any(Error) - ); - }); - - it('should successfully export contacts when service is working', async () => { - const contacts = [ - createMockContact({ email: 'user1@test.com' }), - createMockContact({ email: 'user2@test.com' }) - ]; - - await exporter.export(contacts, mockOptions); - - expect(mockService.people.batchCreateContacts).toHaveBeenCalled(); - - const callArgs = mockService.people.batchCreateContacts.mock.calls[0]; - if (callArgs) { - const contactsCreated = callArgs[0].requestBody?.contacts; - expect(contactsCreated).toHaveLength(2); - } - }); - - it('should handle empty contacts array gracefully', async () => { - await exporter.export([], mockOptions); - expect(google.people).toHaveBeenCalled(); - expect(mockService.people.batchCreateContacts).not.toHaveBeenCalled(); - }); - - it('should pass updateEmptyFieldsOnly flag to session', async () => { - const optionsWithUpdate: ExportOptions = { - googleContactsOptions: { - userId: 'user-123', - accessToken: 'valid_token', - refreshToken: 'valid_refresh', - updateEmptyFieldsOnly: true - } - }; - - mockService.people.connections.list.mockResolvedValue( - createMockRes({ - connections: [ - { - resourceName: 'people/c1', - etag: 'etag-1', - emailAddresses: [{ value: 'test@example.com' }], - metadata: { sources: [{ type: 'CONTACT' }] } - } - ] - }) - ); - - const contact = createMockContact({ email: 'test@example.com' }); - await exporter.export([contact], optionsWithUpdate); - expect(mockService.people.batchUpdateContacts).toHaveBeenCalled(); - }); - - it('should handle network errors gracefully', async () => { - const networkError = new Error('Network timeout'); - - mockService.people.searchContacts?.mockRejectedValue(networkError); - - await expect(exporter.export([], mockOptions)).rejects.toThrow( - 'Network timeout' - ); - - expect(logger.error).toHaveBeenCalled(); - }); -}); - -describe('GoogleContactsSession', () => { - let mockService: MockedPeopleService; - let session: GoogleContactsSession; - const APP_NAME = 'leadminer-test'; - const USER_ID = 'user_123'; - - function createMockRes(data: T): GaxiosResponse { - const response = { - data, - status: 200, - statusText: 'OK', - headers: new Headers(), - config: { - url: 'https://mock.api', - method: 'GET' - } - }; - - return response as unknown as GaxiosResponse; - } - - function createMockContact( - overrides: Partial = {} - ): ContactFrontend { - return { - id: '1', - user_id: USER_ID, - email: 'john@doe.com', - name: 'John Doe', - given_name: 'John', - family_name: 'Doe', - tags: ['Tech'], - telephone: ['123456789'], - ...overrides - }; - } - - function createMockPerson( - resourceName: string, - email: string - ): people_v1.Schema$Person { - return { - resourceName, - etag: `etag-${resourceName}`, - metadata: { sources: [{ type: 'CONTACT' }] }, - emailAddresses: [{ value: email }], - names: [{ givenName: 'Existing', familyName: 'Person' }] - }; - } - - beforeEach(() => { - jest.clearAllMocks(); - - const listMock = jest - .fn() - .mockResolvedValue(createMockRes({ connections: [] })); - - const batchCreateMock = jest - .fn() - .mockResolvedValue(createMockRes({})); - - const batchUpdateMock = jest - .fn() - .mockResolvedValue(createMockRes({})); - - const groupsListMock = jest - .fn() - .mockResolvedValue(createMockRes({ contactGroups: [] })); - - const groupsCreateMock = jest - .fn() - .mockResolvedValue(createMockRes({ resourceName: 'groups/new_id' })); - - mockService = { - people: { - connections: { - list: listMock - }, - batchCreateContacts: batchCreateMock, - batchUpdateContacts: batchUpdateMock - }, - contactGroups: { - list: groupsListMock, - create: groupsCreateMock - } - }; - session = new GoogleContactsSession( - mockService as unknown as people_v1.People, - APP_NAME, - USER_ID - ); - }); - - describe('run() - Complete Flow', () => { - it('should execute the complete sync flow in correct order', async () => { - const existingGroup: people_v1.Schema$ContactGroup = { - resourceName: 'groups/existing', - name: 'ExistingTag' - }; - - mockService.contactGroups.list.mockResolvedValue( - createMockRes({ contactGroups: [existingGroup] }) - ); - - const contact = createMockContact({ - email: 'new@test.com', - tags: ['NewTag'] - }); - - await session.run([contact], false); - - const calls = [ - mockService.contactGroups.list.mock.invocationCallOrder[0], - mockService.contactGroups.create.mock.invocationCallOrder[0], - mockService.people.connections.list.mock.invocationCallOrder[0], - mockService.people.batchCreateContacts.mock.invocationCallOrder[0] - ]; - - for (let i = 1; i < calls.length; i += 1) { - expect(calls[i]).toBeGreaterThan(calls[i - 1]); - } - }); - - it('should handle mixed create and update operations', async () => { - const existingPerson = createMockPerson( - 'people/existing', - 'existing@test.com' - ); - - mockService.people.connections.list.mockResolvedValue( - createMockRes({ connections: [existingPerson] }) - ); - - const contacts = [ - createMockContact({ email: 'existing@test.com' }), - createMockContact({ email: 'new@test.com' }) - ]; - - await session.run(contacts, false); - - expect(mockService.people.batchCreateContacts).toHaveBeenCalledTimes(1); - expect(mockService.people.batchUpdateContacts).toHaveBeenCalledTimes(1); - }); - - it('should not create duplicate labels', async () => { - const existingGroup: people_v1.Schema$ContactGroup = { - resourceName: 'groups/tech', - name: 'Tech' - }; - - mockService.contactGroups.list.mockResolvedValue( - createMockRes({ contactGroups: [existingGroup] }) - ); - - const contacts = [ - createMockContact({ tags: ['Tech'] }), - createMockContact({ tags: ['Tech'] }) - ]; - - await session.run(contacts, false); - - // Should only create APP_NAME label, not Tech (already exists) - expect(mockService.contactGroups.create).toHaveBeenCalledTimes(1); - }); - }); - - describe('Sync Logic (run)', () => { - it('should create new contact when no match exists', async () => { - const contact = createMockContact({ email: 'new@test.com' }); - - await session.run([contact], false); - - expect(mockService.people.batchCreateContacts).toHaveBeenCalledTimes(1); - - const callArgs = mockService.people.batchCreateContacts.mock.calls[0]; - if (!callArgs) { - throw new Error('Expected batchCreateContacts to be called'); - } - - const params = callArgs[0]; - const emailValue = - params.requestBody?.contacts?.[0]?.contactPerson?.emailAddresses?.[0] - ?.value; - expect(emailValue).toBe('new@test.com'); - }); - - it('should match and update existing contact by email', async () => { - const existing = createMockPerson('people/c1', 'match@test.com'); - - mockService.people.connections.list.mockResolvedValue( - createMockRes({ connections: [existing] }) - ); - - const contact = createMockContact({ - email: 'match@test.com', - given_name: 'UpdatedName' - }); - - await session.run([contact], false); - - expect(mockService.people.batchUpdateContacts).toHaveBeenCalledTimes(1); - }); - - it('should match by phone if email match fails', async () => { - const existing: people_v1.Schema$Person = { - ...createMockPerson('people/p1', 'other@test.com'), - phoneNumbers: [{ value: '987654321' }] - }; - - mockService.people.connections.list.mockResolvedValue( - createMockRes({ connections: [existing] }) - ); - - const contact = createMockContact({ - email: 'new@test.com', - telephone: ['987654321'] - }); - - await session.run([contact], false); - - expect(mockService.people.batchUpdateContacts).toHaveBeenCalledTimes(1); - - const callArgs = mockService.people.batchUpdateContacts.mock.calls[0]; - if (!callArgs) { - throw new Error('Expected batchUpdateContacts to be called'); - } - - const params = callArgs[0]; - const contacts = params.requestBody?.contacts; - expect(contacts).toBeDefined(); - expect(contacts?.['people/p1']).toBeDefined(); - }); - - it('should handle labels (App Name + Tags) correctly', async () => { - const contact = createMockContact({ tags: ['OpenSource', 'leads'] }); - - await session.run([contact], false); - - expect(mockService.contactGroups.create).toHaveBeenCalledTimes(3); - - const labelsCreated = mockService.contactGroups.create.mock.calls.map( - (call) => { - const params = call[0]; - return params.requestBody?.contactGroup?.name; - } - ); - - expect(labelsCreated).toContain(APP_NAME); - expect(labelsCreated).toContain('OpenSource'); - expect(labelsCreated).toContain('leads'); - }); - - it('should create new contact when multiple ambiguous matches exist', async () => { - const person1 = createMockPerson('people/p1', 'test@test.com'); - const person2 = createMockPerson('people/p2', 'test@test.com'); - - mockService.people.connections.list.mockResolvedValue( - createMockRes({ connections: [person1, person2] }) - ); - - const contact = createMockContact({ email: 'test@test.com' }); - - await session.run([contact], false); - - // Should create instead of update when ambiguous - expect(mockService.people.batchCreateContacts).toHaveBeenCalledTimes(1); - expect(mockService.people.batchUpdateContacts).not.toHaveBeenCalled(); - }); - }); - - describe('Update Modes (updateEmptyOnly)', () => { - it('should not overwrite existing data when updateEmptyOnly is true', async () => { - const existing: people_v1.Schema$Person = { - ...createMockPerson('people/c1', 'test@test.com'), - organizations: [{ name: 'Old Corp', title: 'CEO' }] - }; - - mockService.people.connections.list.mockResolvedValue( - createMockRes({ connections: [existing] }) - ); - - const incoming = createMockContact({ - email: 'test@test.com', - works_for: 'New Corp' - }); - - await session.run([incoming], true); - - const callArgs = mockService.people.batchUpdateContacts.mock.calls[0]; - if (!callArgs) { - throw new Error('Expected batchUpdateContacts to be called'); - } - - const params = callArgs[0]; - const updatedContact = params.requestBody?.contacts?.['people/c1']; - const orgName = updatedContact?.organizations?.[0]?.name; - - expect(orgName).toBe('Old Corp'); - }); - - it('should overwrite existing data when updateEmptyOnly is false', async () => { - const existing: people_v1.Schema$Person = { - ...createMockPerson('people/c1', 'test@test.com'), - organizations: [{ name: 'Old Corp', title: 'CEO' }] - }; - - mockService.people.connections.list.mockResolvedValue( - createMockRes({ connections: [existing] }) - ); - - const incoming = createMockContact({ - email: 'test@test.com', - works_for: 'New Corp' - }); - - await session.run([incoming], false); - - const callArgs = mockService.people.batchUpdateContacts.mock.calls[0]; - if (!callArgs) { - throw new Error('Expected batchUpdateContacts to be called'); - } - - const params = callArgs[0]; - const updatedContact = params.requestBody?.contacts?.['people/c1']; - const orgName = updatedContact?.organizations?.pop()?.name; - - expect(orgName).toBe('New Corp'); - }); - }); - - describe('mapToPerson', () => { - it('should map all contact fields correctly for new contact', async () => { - const contact = createMockContact({ - email: 'test@example.com', - name: 'John Doe', - given_name: 'John', - family_name: 'Doe', - telephone: ['123456789', '987654321'], - works_for: 'ACME Corp', - job_title: 'Software Engineer', - location: '123 Main St', - same_as: ['https://linkedin.com/in/johndoe'], - tags: ['VIP', 'Client'] - }); - - await session.run([contact], false); - - const callArgs = mockService.people.batchCreateContacts.mock.calls[0]; - if (!callArgs) { - throw new Error('Expected batchCreateContacts to be called'); - } - - const params = callArgs[0]; - const person = params.requestBody?.contacts?.[0]?.contactPerson; - - expect(person?.names?.[0]).toMatchObject({ - givenName: 'John', - familyName: 'Doe', - unstructuredName: 'John Doe' - }); - expect(person?.emailAddresses?.[0]?.value).toBe('test@example.com'); - expect(person?.phoneNumbers).toHaveLength(2); - expect(person?.organizations?.[0]).toMatchObject({ - name: 'ACME Corp', - title: 'Software Engineer' - }); - expect(person?.addresses?.[0]?.streetAddress).toBe('123 Main St'); - expect(person?.urls?.[0]?.value).toBe('https://linkedin.com/in/johndoe'); - }); - - it('should preserve existing values and add new ones when updating', async () => { - const existing: people_v1.Schema$Person = { - ...createMockPerson('people/c1', 'old@test.com'), - emailAddresses: [{ value: 'old@test.com' }], - phoneNumbers: [{ value: '111111111' }] - }; - - mockService.people.connections.list.mockResolvedValue( - createMockRes({ connections: [existing] }) - ); - - const incoming = createMockContact({ - email: 'old@test.com', - telephone: ['222222222'] - }); - - await session.run([incoming], false); - - const callArgs = mockService.people.batchUpdateContacts.mock.calls[0]; - if (!callArgs) { - throw new Error('Expected batchUpdateContacts to be called'); - } - - const params = callArgs[0]; - const person = params.requestBody?.contacts?.['people/c1']; - - // Should have both old and new phone numbers - expect(person?.phoneNumbers).toHaveLength(2); - expect(person?.phoneNumbers?.some((p) => p.value === '111111111')).toBe( - true - ); - expect(person?.phoneNumbers?.some((p) => p.value === '222222222')).toBe( - true - ); - }); - - it('should handle memberships and labels correctly', async () => { - mockService.contactGroups.list.mockResolvedValue( - createMockRes({ - contactGroups: [ - { resourceName: 'groups/app', name: APP_NAME }, - { resourceName: 'groups/vip', name: 'VIP' } - ] - }) - ); - - const existing: people_v1.Schema$Person = { - ...createMockPerson('people/c1', 'test@test.com'), - memberships: [ - { contactGroupMembership: { contactGroupResourceName: 'groups/old' } } - ] - }; - - mockService.people.connections.list.mockResolvedValue( - createMockRes({ connections: [existing] }) - ); - - const contact = createMockContact({ - email: 'test@test.com', - tags: ['VIP'] - }); - - await session.run([contact], false); - - const callArgs = mockService.people.batchUpdateContacts.mock.calls[0]; - if (!callArgs) { - throw new Error('Expected batchUpdateContacts to be called'); - } - - const params = callArgs[0]; - const person = params.requestBody?.contacts?.['people/c1']; - - expect(person?.memberships).toHaveLength(3); // old + app + VIP - expect( - person?.memberships?.some( - (m) => - m.contactGroupMembership?.contactGroupResourceName === 'groups/app' - ) - ).toBe(true); - expect( - person?.memberships?.some( - (m) => - m.contactGroupMembership?.contactGroupResourceName === 'groups/vip' - ) - ).toBe(true); - }); - - it('should deduplicate phone numbers correctly', async () => { - const existing: people_v1.Schema$Person = { - ...createMockPerson('people/c1', 'test@test.com'), - phoneNumbers: [{ value: '123456789' }] - }; - - mockService.people.connections.list.mockResolvedValue( - createMockRes({ connections: [existing] }) - ); - - const incoming = createMockContact({ - email: 'test@test.com', - telephone: ['123456789'] - }); - - await session.run([incoming], false); - - const callArgs = mockService.people.batchUpdateContacts.mock.calls[0]; - if (!callArgs) { - throw new Error('Expected batchUpdateContacts to be called'); - } - - const params = callArgs[0]; - const person = params.requestBody?.contacts?.['people/c1']; - - // Should only have one phone number (deduplicated) - expect(person?.phoneNumbers).toHaveLength(1); - expect(person?.phoneNumbers?.[0]?.value).toBe('123456789'); - }); - - it('should replace existing name when updating (singleton behavior)', async () => { - const existing: people_v1.Schema$Person = { - ...createMockPerson('people/c1', 'test@test.com'), - names: [ - { - givenName: 'Old', - familyName: 'Name', - unstructuredName: 'Old Name' - } - ] - }; - - mockService.people.connections.list.mockResolvedValue( - createMockRes({ connections: [existing] }) - ); - - const incoming = createMockContact({ - email: 'test@test.com', - given_name: 'New', - family_name: 'Name', - name: 'New Name' - }); - - await session.run([incoming], false); - - const callArgs = mockService.people.batchUpdateContacts.mock.calls[0]; - if (!callArgs) { - throw new Error('Expected batchUpdateContacts to be called'); - } - - const params = callArgs[0]; - const person = params.requestBody?.contacts?.['people/c1']; - - // Should have only ONE name (singleton), not append - expect(person?.names).toHaveLength(1); - expect(person?.names?.[0]).toMatchObject({ - givenName: 'New', - familyName: 'Name', - unstructuredName: 'New Name' - }); - }); - - it('should add phone numbers when updateEmptyOnly=true and no existing phones', async () => { - const existing: people_v1.Schema$Person = { - ...createMockPerson('people/c1', 'test@test.com'), - phoneNumbers: [] - }; - - mockService.people.connections.list.mockResolvedValue( - createMockRes({ connections: [existing] }) - ); - - const incoming = createMockContact({ - email: 'test@test.com', - telephone: ['123456789'] - }); - - await session.run([incoming], true); - - const callArgs = mockService.people.batchUpdateContacts.mock.calls[0]; - if (!callArgs) { - throw new Error('Expected batchUpdateContacts to be called'); - } - - const params = callArgs[0]; - const person = params.requestBody?.contacts?.['people/c1']; - - // Should add phone number even with updateEmptyOnly=true when no existing phones - expect(person?.phoneNumbers).toHaveLength(1); - expect(person?.phoneNumbers?.[0]?.value).toBe('123456789'); - }); - - it('should not create name entry with only null values', async () => { - const contact = createMockContact({ - email: 'test@example.com', - name: null as unknown as undefined, - given_name: null as unknown as undefined, - family_name: null as unknown as undefined - }); - - await session.run([contact], false); - - const callArgs = mockService.people.batchCreateContacts.mock.calls[0]; - if (!callArgs) { - throw new Error('Expected batchCreateContacts to be called'); - } - - const params = callArgs[0]; - const person = params.requestBody?.contacts?.[0]?.contactPerson; - - // Should not have names array if all name fields are null/empty - expect(person?.names).toHaveLength(0); - }); - - it('should deduplicate URLs correctly', async () => { - const existing: people_v1.Schema$Person = { - ...createMockPerson('people/c1', 'test@test.com'), - urls: [{ value: 'https://example.com' }] - }; - - mockService.people.connections.list.mockResolvedValue( - createMockRes({ connections: [existing] }) - ); - - const incoming = createMockContact({ - email: 'test@test.com', - same_as: ['https://example.com', 'https://newsite.com'] - }); - - await session.run([incoming], false); - - const callArgs = mockService.people.batchUpdateContacts.mock.calls[0]; - if (!callArgs) { - throw new Error('Expected batchUpdateContacts to be called'); - } - - const params = callArgs[0]; - const person = params.requestBody?.contacts?.['people/c1']; - - // Should have both URLs without duplication - expect(person?.urls).toHaveLength(2); - expect(person?.urls?.some((u) => u.value === 'https://example.com')).toBe( - true - ); - expect(person?.urls?.some((u) => u.value === 'https://newsite.com')).toBe( - true - ); - }); - - it('should deduplicate organizations correctly', async () => { - const existing: people_v1.Schema$Person = { - ...createMockPerson('people/c1', 'test@test.com'), - organizations: [{ name: 'ACME Corp', title: 'Engineer' }] - }; - - mockService.people.connections.list.mockResolvedValue( - createMockRes({ connections: [existing] }) - ); - - const incoming = createMockContact({ - email: 'test@test.com', - works_for: 'ACME Corp', - job_title: 'Engineer' - }); - - await session.run([incoming], false); - - const callArgs = mockService.people.batchUpdateContacts.mock.calls[0]; - if (!callArgs) { - throw new Error('Expected batchUpdateContacts to be called'); - } - - const params = callArgs[0]; - const person = params.requestBody?.contacts?.['people/c1']; - - // Should have only one organization (deduplicated) - expect(person?.organizations).toHaveLength(1); - expect(person?.organizations?.[0]).toMatchObject({ - name: 'ACME Corp', - title: 'Engineer' - }); - }); - }); - - describe('System Safety & Pagination', () => { - it('should handle large batches by chunking contacts into groups of 200', async () => { - const manyContacts = Array.from({ length: 250 }, (_, i) => - createMockContact({ email: `test${i}@test.com` }) - ); - - await session.run(manyContacts, false); - - expect(mockService.people.batchCreateContacts).toHaveBeenCalledTimes(2); - - // First batch should have 200 contacts - const firstCall = mockService.people.batchCreateContacts.mock.calls[0]; - if (firstCall) { - const firstBatchSize = firstCall[0].requestBody?.contacts?.length; - expect(firstBatchSize).toBe(200); - } - - // Second batch should have 50 contacts - const secondCall = mockService.people.batchCreateContacts.mock.calls[1]; - if (secondCall) { - const secondBatchSize = secondCall[0].requestBody?.contacts?.length; - expect(secondBatchSize).toBe(50); - } - }); - - it('should follow pagination for large contact lists', async () => { - mockService.people.connections.list - .mockResolvedValueOnce( - createMockRes({ - connections: [createMockPerson('p1', '1@a.com')], - nextPageToken: 'page2' - }) - ) - .mockResolvedValueOnce( - createMockRes({ - connections: [createMockPerson('p2', '2@a.com')] - }) - ); - - await session.run([createMockContact({ email: 'new@a.com' })], false); - - expect(mockService.people.connections.list).toHaveBeenCalledTimes(2); - - const secondCallArgs = mockService.people.connections.list.mock.calls[1]; - if (!secondCallArgs) { - throw new Error('Expected second call to connections.list'); - } - - const params = secondCallArgs[0]; - expect(params.pageToken).toBe('page2'); - }); - - it('should handle pagination with multiple pages', async () => { - mockService.people.connections.list - .mockResolvedValueOnce( - createMockRes({ - connections: [createMockPerson('p1', '1@a.com')], - nextPageToken: 'page2' - }) - ) - .mockResolvedValueOnce( - createMockRes({ - connections: [createMockPerson('p2', '2@a.com')], - nextPageToken: 'page3' - }) - ) - .mockResolvedValueOnce( - createMockRes({ - connections: [createMockPerson('p3', '3@a.com')] - }) - ); - - await session.run([createMockContact({ email: 'new@a.com' })], false); - - expect(mockService.people.connections.list).toHaveBeenCalledTimes(3); - }); - }); - - describe('Edge Cases', () => { - it('should handle empty contact list gracefully', async () => { - await session.run([], false); - - expect(mockService.people.batchCreateContacts).not.toHaveBeenCalled(); - expect(mockService.people.batchUpdateContacts).not.toHaveBeenCalled(); - }); - - it('should handle contacts without phone numbers', async () => { - const contact = createMockContact({ - email: 'test@test.com', - telephone: undefined - }); - - await session.run([contact], false); - - const callArgs = mockService.people.batchCreateContacts.mock.calls[0]; - if (callArgs) { - const person = callArgs[0].requestBody?.contacts?.[0]?.contactPerson; - expect(person?.phoneNumbers).toEqual([]); - } - }); - - it('should handle contacts without tags', async () => { - const contact = createMockContact({ - email: 'test@test.com', - tags: undefined - }); - - await session.run([contact], false); - - // Should still create APP_NAME label - expect(mockService.contactGroups.create).toHaveBeenCalledTimes(1); - }); - - it('should normalize phone numbers for matching (remove spaces)', async () => { - const existing: people_v1.Schema$Person = { - ...createMockPerson('people/p1', 'other@test.com'), - phoneNumbers: [{ value: '123 456 789' }] - }; - - mockService.people.connections.list.mockResolvedValue( - createMockRes({ connections: [existing] }) - ); - - const contact = createMockContact({ - email: 'new@test.com', - telephone: ['123456789'] - }); - - await session.run([contact], false); - - // Should match and update - expect(mockService.people.batchUpdateContacts).toHaveBeenCalledTimes(1); - }); - - it('should handle case-insensitive email matching', async () => { - const existing = createMockPerson('people/c1', 'Test@Example.COM'); - - mockService.people.connections.list.mockResolvedValue( - createMockRes({ connections: [existing] }) - ); - - const contact = createMockContact({ - email: 'test@example.com' // Lowercase - }); - - await session.run([contact], false); - - // Should match and update (not create) - expect(mockService.people.batchUpdateContacts).toHaveBeenCalledTimes(1); - expect(mockService.people.batchCreateContacts).not.toHaveBeenCalled(); - }); - }); -}); diff --git a/backend/test/unit/csvHelpers.test.ts b/backend/test/unit/csvHelpers.test.ts deleted file mode 100644 index 5cae8dc17..000000000 --- a/backend/test/unit/csvHelpers.test.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { describe, expect, it, test } from '@jest/globals'; -import CsvExport from '../../src/services/export/exports/csv'; - -describe('getLocalizedCsvSeparator', () => { - test.each([ - ['fr-FR', ';'], - ['de-DE', ';'], - ['es-ES', ';'], - ['pt-PT', ';'], - ['it-IT', ';'], - ['en-US', ','], - ['ja-JP', ','] - ])('should return ; for language %s', (language, expectedSeparator) => { - expect(CsvExport.getLocalizedCsvSeparator(language)).toBe( - expectedSeparator - ); - }); - - it('should return default for falsy strings', () => { - expect(CsvExport.getLocalizedCsvSeparator('')).toBe(','); - expect(CsvExport.getLocalizedCsvSeparator('test')).toBe(','); - }); -}); - -describe('getCsvStr', () => { - it('should generate a CSV string with the specified delimiter', async () => { - const delimiter = ','; - const rows = [ - { name: 'Alice', age: 30 }, - { name: 'Bob', age: 25 } - ]; - const csvString = await CsvExport.getCsvStr( - [ - { key: 'name', header: 'Name' }, - { key: 'age', header: 'Age' } - ], - rows, - delimiter - ); - - expect(csvString).toContain('"Name","Age"'); - expect(csvString).toContain('"Alice",30'); - expect(csvString).toContain('"Bob",25'); - }); -}); diff --git a/backend/test/unit/enrichment-service/enrichLayer/client.test.ts b/backend/test/unit/enrichment-service/enrichLayer/client.test.ts deleted file mode 100644 index b901cfdb0..000000000 --- a/backend/test/unit/enrichment-service/enrichLayer/client.test.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { beforeEach, describe, expect, it, jest } from '@jest/globals'; - -import axios from 'axios'; -import { Logger } from 'winston'; -import EnrichLayerAPI, { - ReverseEmailLookupParams, - ReverseEmailLookupResponse -} from '../../../../src/services/enrichment/enrich-layer/client'; -import { - Distribution, - TokenBucketRateLimiter -} from '../../../../src/services/rate-limiter'; - -jest.mock('axios'); - -jest.mock('ioredis'); - -jest.mock('../../../../src/config', () => ({ - LEADMINER_API_LOG_LEVEL: 'debug' -})); - -const mockLogger: Logger = { - info: jest.fn(), - error: jest.fn() -} as unknown as Logger; - -const mockAxiosInstance = { - get: jest.fn() -}; -(axios.create as jest.Mock).mockReturnValue(mockAxiosInstance); - -const ENRICH_LAYER_THROTTLE_REQUESTS = 5; -const ENRICH_LAYER_THROTTLE_INTERVAL = 0.1; - -const RATE_LIMITER = new TokenBucketRateLimiter({ - executeEvenly: true, - uniqueKey: 'email_verification_enrichLayer_test', - distribution: Distribution.Memory, - requests: ENRICH_LAYER_THROTTLE_REQUESTS, - intervalSeconds: ENRICH_LAYER_THROTTLE_INTERVAL -}); - -describe('EnrichLayerAPI', () => { - const config = { - url: 'https://api.example.com', - apiKey: 'dummy-api-key', - rateLimiter: RATE_LIMITER - }; - - const enrichLayer = new EnrichLayerAPI(config, mockLogger); - - beforeEach(() => { - jest.clearAllMocks(); - }); - - describe('reverseEmailLookup', () => { - it('should retry on rate limit error', async () => { - const params: ReverseEmailLookupParams = { - email: 'test@example.com', - lookup_depth: 'deep' - }; - - const rateLimitError = { response: { status: 429 } }; - const successResponse = { - data: { email: params.email } as ReverseEmailLookupResponse - }; - - mockAxiosInstance.get - .mockImplementationOnce(() => Promise.reject(rateLimitError)) - .mockImplementationOnce(() => Promise.reject(rateLimitError)) - .mockImplementationOnce(() => Promise.resolve(successResponse)); - - const response = await enrichLayer.reverseEmailLookup(params); - - expect(mockAxiosInstance.get).toHaveBeenCalledTimes(3); // Initial attempt + 2 retries - expect(mockLogger.info).toHaveBeenCalledWith( - '[EnrichLayerAPI:rateLimitRetryWithExponentialBackoff] Rate limited, retrying attempt 1 waiting 2000ms before retry' - ); - expect(mockLogger.info).toHaveBeenCalledWith( - '[EnrichLayerAPI:rateLimitRetryWithExponentialBackoff] Rate limited, retrying attempt 2 waiting 4000ms before retry' - ); - expect(response.email).toBe(params.email); - }, 10000); - - it('should not retry on non-rate limit error', async () => { - const params: ReverseEmailLookupParams = { - email: 'test@example.com', - lookup_depth: 'deep' - }; - - mockAxiosInstance.get.mockImplementationOnce(() => - // eslint-disable-next-line prefer-promise-reject-errors - Promise.reject({ response: { status: 500 } }) - ); - - await expect(enrichLayer.reverseEmailLookup(params)).rejects.toThrow(); - expect(mockAxiosInstance.get).toHaveBeenCalledTimes(1); - expect(mockLogger.info).not.toHaveBeenCalledWith( - '[EnrichLayerAPI:rateLimitRetryWithExponentialBackoff] Rate limited, retrying attempt 1 waiting 2000ms before retry' - ); - }); - - it('should properly use rate limiter', async () => { - const email = 'test@example.com'; - const params: ReverseEmailLookupParams = { - email, - lookup_depth: 'superficial', - enrich_profile: 'skip' - }; - mockAxiosInstance.get.mockReturnValue({ - data: { - email, - profile: {}, - last_updated: '', - similarity_score: 0, - linkedin_profile_url: '', - facebook_profile_url: '', - twitter_profile_url: '' - } - }); - - const startTime = Date.now(); - const requests = Array.from({ - length: ENRICH_LAYER_THROTTLE_REQUESTS - }).map(() => enrichLayer.reverseEmailLookup(params)); - await Promise.all(requests); - const totalTime = Date.now() - startTime; - - expect(totalTime).toBeGreaterThanOrEqual( - ENRICH_LAYER_THROTTLE_INTERVAL * (requests.length - 1) - ); - }); - }); -}); diff --git a/backend/test/unit/enrichment-service/enrichLayer/enricher.test.ts b/backend/test/unit/enrichment-service/enrichLayer/enricher.test.ts deleted file mode 100644 index a94ea7606..000000000 --- a/backend/test/unit/enrichment-service/enrichLayer/enricher.test.ts +++ /dev/null @@ -1,357 +0,0 @@ -import { beforeEach, describe, expect, it, jest } from '@jest/globals'; - -import { Logger } from 'winston'; -import EnrichLayerAPI, { - ProfileExtra, - ReverseEmailLookupResponse -} from '../../../../src/services/enrichment/enrich-layer/client'; -import { Person } from '../../../../src/db/types'; -import EnrichLayer from '../../../../src/services/enrichment/enrich-layer'; - -describe('EnrichLayer', () => { - let mockClient: jest.Mocked; - let mockLogger: jest.Mocked; - let enricher: EnrichLayer; - - beforeEach(() => { - mockClient = { - reverseEmailLookup: jest.fn() - } as Partial> as jest.Mocked; - - mockLogger = { - debug: jest.fn(), - info: jest.fn(), - warn: jest.fn(), - error: jest.fn() - } as unknown as jest.Mocked; - - enricher = new EnrichLayer(mockClient, mockLogger); - }); - - describe('getProfileUrls', () => { - it('should return correct URLs based on ProfileExtra properties', () => { - const profile: ProfileExtra = { - github_profile_id: 'user123', - facebook_profile_id: 'user.fb', - twitter_profile_id: 'user_twitter', - website: 'https://example.com' - }; - - const urls = EnrichLayer.getProfileUrls(profile); - - expect(urls).toEqual([ - 'https://github.com/user123', - 'https://facebook.com/user.fb', - 'https://twitter.com/user_twitter', - 'https://example.com' - ]); - }); - - it('should return an empty array if no ProfileExtra properties are provided', () => { - const profile: ProfileExtra = {}; - const urls = EnrichLayer.getProfileUrls(profile); - - expect(urls).toEqual([]); - }); - }); - - describe('parseResult', () => { - const enrichResultMock: ReverseEmailLookupResponse[] = [ - { - email: 'valid@example.com', - profile: { - full_name: 'John Doe', - first_name: 'John', - last_name: 'Doe', - occupation: 'Software Engineer', - profile_pic_url: 'https://example.com/pic.jpg', - public_identifier: 'john.doe', - extra: { - website: 'https://johndoe.com' - }, - // Required fields for the Profile type - city: 'New York', - state: 'NY', - country: 'US', - country_full_name: 'United States', - languages: ['English'], - experiences: [ - { - starts_at: { day: 1, month: 1, year: 2020 }, - ends_at: { day: 1, month: 1, year: 2022 }, - company: 'Tech Inc.', - title: 'Developer', - description: 'Worked on various projects' - } - ] - }, - last_updated: '2024-11-01', - similarity_score: 0.9, - linkedin_profile_url: 'https://linkedin.com/in/johndoe', - facebook_profile_url: 'https://facebook.com/johndoe', - twitter_profile_url: 'https://twitter.com/johndoe' - }, - { - email: 'partial@example.com', - profile: { - full_name: 'Partial User', - first_name: '', - last_name: 'User', - occupation: 'Software Engineer', - profile_pic_url: 'https://example.com/partialuser.jpg', - public_identifier: 'partial.user', - extra: { - website: 'https://partialuser.com' - }, - // Required fields for the Profile type - city: '', - state: '', - country: '', - country_full_name: '', - languages: [], - experiences: [] - }, - last_updated: '2024-11-02', - similarity_score: 0.7, - linkedin_profile_url: 'https://linkedin.com/in/partialuser', - facebook_profile_url: 'https://facebook.com/partialuser', - twitter_profile_url: 'https://twitter.com/partialuser' - }, - { - email: 'partial2@example.com', - profile: { - full_name: 'Partial2 User', - first_name: 'Partial2', - last_name: 'User', - occupation: '', - profile_pic_url: '', - public_identifier: 'partial2.user', - extra: {}, - // Required fields for the Profile type - city: 'city', - state: 'state', - country: 'country', - country_full_name: '', - languages: [], - experiences: [] - }, - last_updated: '2024-11-03', - similarity_score: 0.5, - linkedin_profile_url: 'https://linkedin.com/in/partial2user', - facebook_profile_url: '', - twitter_profile_url: '' - }, - { - email: 'invalid@example.com', - profile: { - full_name: '', - first_name: '', - last_name: '', - occupation: '', - profile_pic_url: '', - public_identifier: '', - extra: {}, - // Required fields for the Profile type - city: '', - state: '', - country: '', - country_full_name: '', - languages: [], - experiences: [] - }, - last_updated: '2024-11-03', - similarity_score: 0.5, - linkedin_profile_url: '', - facebook_profile_url: '', - twitter_profile_url: '' - } - ]; - it('should map full ReverseEmailLookupResponse data correctly', () => { - const result = enricher.parseResult([enrichResultMock[0]]); - - expect(result).toEqual({ - engine: 'enrichLayer', - data: [ - { - email: 'valid@example.com', - name: 'John Doe', - givenName: 'John', - familyName: 'Doe', - jobTitle: 'Software Engineer', - organization: 'Tech Inc.', - image: 'https://example.com/pic.jpg', - identifiers: ['john.doe'], - location: 'New York, NY, United States', - sameAs: [ - 'https://linkedin.com/in/johndoe', - 'https://facebook.com/johndoe', - 'https://twitter.com/johndoe', - 'https://johndoe.com' - ] - } - ], - raw_data: [enrichResultMock[0]] - }); - }); - - it('should map partial ReverseEmailLookupResponse data correctly', () => { - let result = enricher.parseResult([enrichResultMock[1]]); - - expect(result).toEqual({ - engine: 'enrichLayer', - data: [ - { - email: 'partial@example.com', - name: 'Partial User', - familyName: 'User', - jobTitle: 'Software Engineer', - identifiers: ['partial.user'], - sameAs: [ - 'https://linkedin.com/in/partialuser', - 'https://facebook.com/partialuser', - 'https://twitter.com/partialuser', - 'https://partialuser.com' - ], - image: 'https://example.com/partialuser.jpg', - location: undefined, - givenName: undefined, - organization: undefined, - githubProfile: undefined, - alternateName: undefined - } - ], - raw_data: [enrichResultMock[1]] - }); - - result = enricher.parseResult([enrichResultMock[2]]); - - expect(result).toEqual({ - engine: 'enrichLayer', - data: [ - { - email: 'partial2@example.com', - name: 'Partial2 User', - givenName: 'Partial2', - familyName: 'User', - identifiers: ['partial2.user'], - sameAs: ['https://linkedin.com/in/partial2user'], - location: 'city, state', - image: undefined, - jobTitle: undefined, - organization: undefined, - alternateName: undefined - } - ], - raw_data: [enrichResultMock[2]] - }); - }); - - it('should map invalid/empty ReverseEmailLookupResponse data correctly', () => { - const result = enricher.parseResult([enrichResultMock[3]]); - - expect(result).toEqual({ - engine: 'enrichLayer', - data: [], - raw_data: [enrichResultMock[3]] - }); - }); - }); - - describe('enrichSync', () => { - it('should call reverseEmailLookup and map the response correctly', async () => { - const person: Partial = { email: 'test@example.com' }; - const mockResponse: ReverseEmailLookupResponse = { - email: 'test@example.com', - profile: { - city: 'New York', - full_name: 'John Doe', - first_name: 'John', - last_name: 'Doe', - state: 'NY', - country: 'US', - country_full_name: 'United States', - languages: ['English'], - occupation: 'Software Engineer', - profile_pic_url: 'https://example.com/pic.jpg', - public_identifier: 'john.doe', - extra: { - github_profile_id: 'johndoe', - website: 'https://johndoe.com' - }, - experiences: [ - { - starts_at: { day: 1, month: 1, year: 2020 }, - ends_at: { day: 1, month: 1, year: 2022 }, - company: 'Tech Inc.', - title: 'Developer', - description: 'Worked on various projects' - } - ] - }, - last_updated: '2024-11-01', - similarity_score: 0.9, - linkedin_profile_url: 'https://linkedin.com/in/johndoe', - facebook_profile_url: 'https://facebook.com/johndoe', - twitter_profile_url: 'https://twitter.com/johndoe' - }; - - mockClient.reverseEmailLookup.mockResolvedValue(mockResponse); - - const result = await enricher.enrichSync(person); - - expect(mockClient.reverseEmailLookup).toHaveBeenCalledWith({ - email: 'test@example.com', - lookup_depth: 'superficial', - enrich_profile: 'enrich' - }); - expect(result).toEqual({ - engine: 'enrichLayer', - data: [ - { - email: 'test@example.com', - name: 'John Doe', - givenName: 'John', - familyName: 'Doe', - jobTitle: 'Software Engineer', - organization: 'Tech Inc.', - image: 'https://example.com/pic.jpg', - identifiers: ['john.doe'], - location: 'New York, NY, United States', - sameAs: [ - 'https://linkedin.com/in/johndoe', - 'https://facebook.com/johndoe', - 'https://twitter.com/johndoe', - 'https://github.com/johndoe', - 'https://johndoe.com' - ] - } - ], - raw_data: [mockResponse] - }); - }); - - it('should throw an error if reverseEmailLookup fails', async () => { - const person: Partial = { email: 'test@example.com' }; - const error = new Error('Network error'); - mockClient.reverseEmailLookup.mockRejectedValue(error); - - await expect(enricher.enrichSync(person)).rejects.toThrow( - 'Network error' - ); - }); - }); - - describe('enrichAsync', () => { - it('should throw a "not implemented" error', async () => { - try { - await enricher.enrichAsync( - [{ email: 'test@example.com' }], - 'webhook-url' - ); - } catch (error) { - expect(error instanceof Error).toBeTruthy(); - expect((error as Error).message).toEqual('Method not implemented.'); - } - }); - }); -}); diff --git a/backend/test/unit/enrichment-service/enricher.test.ts b/backend/test/unit/enrichment-service/enricher.test.ts deleted file mode 100644 index 00243895e..000000000 --- a/backend/test/unit/enrichment-service/enricher.test.ts +++ /dev/null @@ -1,234 +0,0 @@ -import { afterEach, describe, expect, it, jest } from '@jest/globals'; - -import { Contact } from '../../../src/db/types'; -import { Engine } from '../../../src/services/enrichment/Engine'; -import Enricher from '../../../src/services/enrichment/Enricher'; -import logger from '../../../src/utils/logger'; - -jest.mock('../../../src/config', () => ({ - LEADMINER_API_HOST: 'leadminer-test.io/api/enrich/webhook', - LEADMINER_API_LOG_LEVEL: 'error', - SUPABASE_PROJECT_URL: 'fake', - SUPABASE_SECRET_PROJECT_TOKEN: 'fake' -})); - -jest.mock('../../../src/utils/logger'); - -jest.mock('@supabase/supabase-js'); -jest.mock('../../../src/utils/supabase', () => ({ - from: jest.fn().mockReturnThis(), - schema: jest.fn().mockReturnThis(), - select: jest.fn(), - insert: jest.fn(), - upsert: jest.fn(), - update: jest.fn(), - delete: jest.fn(), - rpc: jest.fn() -})); - -function createMockEnricher(name: string, isSync: boolean, isAsync: boolean) { - return { - name, - isSync, - isAsync, - isValid: jest.fn(), - enrichSync: jest.fn(), - enrichAsync: jest.fn(), - parseResult: jest.fn() - } as jest.Mocked; -} - -describe('Enricher Class', () => { - const mockEngineSync1 = createMockEnricher('engine-sync-1', true, false); - const mockEngineSync2 = createMockEnricher('engine-sync-2', true, false); - const mockEngineAsync = createMockEnricher('engine-async-1', false, true); - - const enricher = new Enricher( - [mockEngineSync1, mockEngineSync2, mockEngineAsync], - logger - ); - - afterEach(() => { - jest.clearAllMocks(); - }); - - describe('enrichSync', () => { - it('should use the first valid synchronous engine to enrich a contact', async () => { - const contact: Partial = { email: 'test@example.com' }; - const engineResult = { - engine: mockEngineSync1.name, - data: [{ email: 'test@example.com', name: 'Test User' }], - raw_data: [{ email: 'test@example.com', name: 'Test User' }] - }; - mockEngineSync1.isValid.mockReturnValue(true); - mockEngineSync1.enrichSync.mockResolvedValue(engineResult); - - const result = await enricher.enrichSync(contact); - - expect(mockEngineSync1.isValid).toHaveBeenCalledWith(contact); - expect(mockEngineSync1.enrichSync).toHaveBeenCalledWith(contact); - expect(result).toEqual(engineResult); - }); - - it('should pass to the next engine if the first is not valid', async () => { - const contact: Partial = { email: 'test@example.com' }; - const engineResult = { - engine: mockEngineSync2.name, - data: [{ email: 'test@example.com', name: 'Test User' }], - raw_data: [{ email: 'test@example.com', name: 'Test User' }] - }; - - mockEngineSync1.isValid.mockReturnValue(false); // First engine invalid - mockEngineSync2.isValid.mockReturnValue(true); // Second engine valid - mockEngineSync2.enrichSync.mockResolvedValue(engineResult); - - const result = await enricher.enrichSync(contact); - - expect(mockEngineSync1.isValid).toHaveBeenCalledWith(contact); - expect(mockEngineSync2.isValid).toHaveBeenCalledWith(contact); - expect(mockEngineSync2.enrichSync).toHaveBeenCalledWith(contact); - expect(result).toEqual(engineResult); - }); - - it('should pass to the next engine if the first engine returns no result', async () => { - const contact: Partial = { email: 'test@example.com' }; - const engineResult = { - engine: mockEngineSync2.name, - data: [{ email: 'test@example.com', name: 'Test User' }], - raw_data: [{ email: 'test@example.com', name: 'Test User' }] - }; - - mockEngineSync1.isValid.mockReturnValue(true); // First engine valid - mockEngineSync1.enrichSync.mockResolvedValue({ - engine: mockEngineSync1.name, - data: [], - raw_data: [] - }); // First engine returns no result - mockEngineSync2.isValid.mockReturnValue(true); // Second engine valid - mockEngineSync2.enrichSync.mockResolvedValue(engineResult); - - const result = await enricher.enrichSync(contact); - - expect(mockEngineSync1.isValid).toHaveBeenCalledWith(contact); - expect(mockEngineSync1.enrichSync).toHaveBeenCalledWith(contact); - expect(mockEngineSync2.isValid).toHaveBeenCalledWith(contact); - expect(mockEngineSync2.enrichSync).toHaveBeenCalledWith(contact); - expect(result).toEqual(engineResult); - }); - - it('should throw an error if no valid engines are available', async () => { - const contact: Partial = { email: 'test@example.com' }; - mockEngineSync1.isValid.mockReturnValue(false); - mockEngineSync2.isValid.mockReturnValue(false); - - await expect(enricher.enrichSync(contact)).rejects.toThrow( - 'No Engines to use.' - ); - }); - }); - - describe('enrichAsync', () => { - it('should use the first valid asynchronous engine to enrich contacts', async () => { - const contacts: Partial[] = [ - { email: 'test@example.com' }, - { email: 'test2@example.com' } - ]; - const webhook = 'http://example.com/webhook'; - const engineResult = { - engine: mockEngineAsync.name, - token: 'mockToken', - data: [], - raw_data: [] - }; - - mockEngineAsync.enrichAsync.mockResolvedValue(engineResult); - - const result = await enricher.enrichAsync(contacts, webhook); - - expect(mockEngineAsync.enrichAsync).toHaveBeenCalledWith( - contacts, - webhook - ); - expect(result).toEqual(engineResult); - }); - - it('should throw an error if no asynchronous engines are available', async () => { - const emptyEnricher = new Enricher( - [createMockEnricher('engine', false, false)], - logger - ); - await expect( - emptyEnricher.enrichAsync([], 'http://example.com/webhook') - ).rejects.toThrow('No Engines to use.'); - }); - }); - - describe('enrich', () => { - it('should yield enriched results for each contact', async () => { - const contacts: Partial[] = [ - { email: 'test1@example.com' }, - { email: 'test2@example.com' } - ]; - const enricherResult = [ - { - engine: mockEngineSync1.name, - data: [{ email: 'test1@example.com', name: 'Test User' }], - raw_data: [{ email: 'test1@example.com', name: 'Test User' }] - }, - { - engine: mockEngineSync2.name, - data: [{ email: 'test2@example.com', name: 'Test User 2' }], - raw_data: [{ email: 'test2@example.com', name: 'Test User 2' }] - } - ]; - - mockEngineSync1.isValid.mockReturnValue(true); - mockEngineSync1.enrichSync - .mockResolvedValueOnce(enricherResult[0]) - .mockResolvedValueOnce({ - engine: mockEngineSync1.name, - data: [], - raw_data: [] - }); - mockEngineSync2.isValid.mockReturnValue(true); - mockEngineSync2.enrichSync.mockResolvedValueOnce(enricherResult[1]); - - const results = []; - for await (const result of enricher.enrich(contacts)) { - results.push(result); - } - - expect(mockEngineSync1.enrichSync).toHaveBeenCalledTimes(2); - expect(mockEngineSync2.enrichSync).toHaveBeenCalledTimes(1); - expect(results).toEqual(enricherResult); - }); - }); - - describe('parseResult', () => { - it('should parse results using the specified engine', () => { - const result = { - engine: mockEngineSync1.name, - data: [{ email: 'parsed@example.com' }], - raw_data: [{ email: 'parsed@example.com' }] - }; - - mockEngineSync1.parseResult.mockReturnValue(result); - - const parsedResult = enricher.parseResult([result], mockEngineSync1.name); - - expect(mockEngineSync1.parseResult).toHaveBeenCalledWith([result]); - expect(parsedResult).toEqual({ - data: result.data, - raw_data: result.raw_data - }); - }); - - it('should return empty arrays if the engine is not found', () => { - const result = [{ key: 'value' }]; - - const parsedResult = enricher.parseResult(result, 'NonExistentEngine'); - - expect(parsedResult).toEqual({ data: [], raw_data: [] }); - }); - }); -}); diff --git a/backend/test/unit/enrichment-service/helpers.test.ts b/backend/test/unit/enrichment-service/helpers.test.ts deleted file mode 100644 index 870342cff..000000000 --- a/backend/test/unit/enrichment-service/helpers.test.ts +++ /dev/null @@ -1,428 +0,0 @@ -import { beforeEach, describe, expect, it, jest } from '@jest/globals'; -import { - EnrichmentCache, - enrichFromCache, - enrichPersonAsync, - enrichPersonSync, - getEnrichmentCache -} from '../../../src/controllers/enrichment.helpers'; - -import { Contact } from '../../../src/db/types'; -import ENV from '../../../src/config'; -import EnrichmentService from '../../../src/services/enrichment'; -import Enrichments from '../../../src/db/supabase/enrichments'; -import logger from '../../../src/utils/logger'; -import supabaseClient from '../../../src/utils/supabase'; - -jest.mock('../../../src/config', () => ({ - LEADMINER_API_HOST: 'leadminer-test.io/api/enrich/webhook', - LEADMINER_API_LOG_LEVEL: 'error', - SUPABASE_PROJECT_URL: 'fake', - SUPABASE_SECRET_PROJECT_TOKEN: 'fake' -})); - -jest.mock('../../../src/utils/logger'); - -jest.mock('@supabase/supabase-js'); -jest.mock('../../../src/utils/supabase', () => ({ - from: jest.fn().mockReturnThis(), - schema: jest.fn().mockReturnThis(), - select: jest.fn(), - insert: jest.fn(), - upsert: jest.fn(), - update: jest.fn(), - delete: jest.fn(), - rpc: jest.fn() -})); - -jest.mock('../../../src/services/enrichment', () => ({ - enrich: jest.fn(), - enrichSync: jest.fn(), - enrichAsync: jest.fn(), - parseResult: jest.fn() -})); - -function mockEngagementsDB() { - return { - task: jest.fn(), - tasks: jest.fn(), - engagements: jest.fn(), - client: jest.fn(), - enrich: jest.fn() - } as unknown as Enrichments; -} - -function mockEnrichmentsDB() { - return { - redactedTask: jest.fn(), - enrich: jest.fn() - } as unknown as Enrichments; -} - -const getCache = jest.fn() as jest.MockedFunction; - -describe('enrichFromCache', () => { - const enrichmentsDB = mockEngagementsDB(); - beforeEach(() => { - jest.clearAllMocks(); - }); - - it('should enrich contacts from cache and return un-enriched contacts', async () => { - const contacts: Contact[] = [ - { id: '1', user_id: 'leadminer-1', email: 'test1@example.com' }, - { id: '2', user_id: 'leadminer-1', email: 'test2@example.com' }, - { id: '3', user_id: 'leadminer-1', email: 'test3@example.com' } - ]; - - const cached = [ - { - engine: 'cache', - data: [ - { - name: 'leadminer', - jobTitle: 'data miner', - email: 'test1@example.com' - }, - { - name: 'leadminer', - jobTitle: 'data miner', - email: 'test2@example.com' - } - ], - raw_data: [ - { - name: 'leadminer', - jobTitle: 'data miner', - email: 'test1@example.com' - }, - { - name: 'leadminer', - jobTitle: 'data miner', - email: 'test2@example.com' - } - ] - } - ]; - - getCache.mockResolvedValue(cached); - - const result = await enrichFromCache(getCache, enrichmentsDB, contacts); - expect(getCache).toHaveBeenCalledWith(contacts, EnrichmentService); - expect(enrichmentsDB.enrich).toHaveBeenCalledWith(cached); - expect(logger.debug).toHaveBeenCalledWith('Enriched from cache.', [ - contacts[0].email, - contacts[1].email - ]); - expect(result).toEqual([contacts[2]]); - }); - - it('should return all contacts if no cached data is found', async () => { - const contacts: Contact[] = [ - { id: '1', user_id: 'leadminer-1', email: 'test1@example.com' }, - { id: '2', user_id: 'leadminer-1', email: 'test2@example.com' } - ]; - - getCache.mockResolvedValue([{} as any]); - - const result = await enrichFromCache(getCache, enrichmentsDB, contacts); - - expect(getCache).toHaveBeenCalledWith(contacts, expect.anything()); - expect(enrichmentsDB.enrich).not.toHaveBeenCalled(); - expect(logger.debug).not.toHaveBeenCalled(); - expect(result).toEqual(contacts); - }); - - it('should handle contacts with missing email field gracefully', async () => { - const contacts: Partial[] = [ - { id: '1', user_id: 'leadminer-1', email: 'test1@example.com' }, - { id: '2', user_id: 'leadminer-1', email: undefined }, - { id: '3', user_id: 'leadminer-1', email: undefined } - ]; - - const cached = [ - { - engine: 'cache', - data: [ - { - name: 'leadminer', - jobTitle: 'data miner', - email: 'test1@example.com' - } - ], - raw_data: [ - { - name: 'leadminer', - jobTitle: 'data miner', - email: 'test1@example.com' - } - ] - } - ]; - - getCache.mockResolvedValue(cached); - - const result = await enrichFromCache(getCache, enrichmentsDB, contacts); - - expect(getCache).toHaveBeenCalledWith(contacts, expect.anything()); - expect(enrichmentsDB.enrich).toHaveBeenCalledWith(cached); - expect(logger.debug).toHaveBeenCalledWith('Enriched from cache.', [ - contacts[0].email - ]); - expect(result).toEqual([]); - }); -}); - -describe('getEnrichmentCache', () => { - beforeEach(() => { - jest.clearAllMocks(); - }); - - it('should retrieve and process enrichment cache successfully', async () => { - const contacts: Partial[] = [ - { email: 'test1@example.com' }, - { email: 'test2@example.com' } - ]; - - const expectedRpcData: EnrichmentCache[] = [ - { - task_id: '', - user_id: '', - created_at: '', - result: { name: 'leadminer', email: 'test1@example.com' }, - engine: 'testEngine' - } - ]; - const expectedParse = [ - { - engine: 'testing', - data: [ - { - name: 'leadminer', - email: 'test1@example.com', - engine: 'testEngines' - } - ], - raw_data: [] - } - ]; - const expectedResult = [ - { - engine: 'cache', - data: [ - { - name: 'leadminer', - email: 'test1@example.com', - engine: 'testEngines' - } - ], - raw_data: [] - } - ]; - - (supabaseClient.rpc as jest.Mock).mockReturnValueOnce({ - data: expectedRpcData, - error: null - }); - (EnrichmentService.parseResult as jest.Mock).mockReturnValueOnce( - expectedParse - ); - - const result = await getEnrichmentCache(contacts, EnrichmentService); - - expect(supabaseClient.rpc as jest.Mock).toHaveBeenCalledWith( - 'enriched_most_recent', - { - emails: ['test1@example.com', 'test2@example.com'] - } - ); - expect(EnrichmentService.parseResult).toHaveBeenCalledWith( - [expectedRpcData[0].result], - expectedRpcData[0].engine - ); - expect(result).toEqual(expectedResult); - }); - - it('should return an empty array when no data is retrieved', async () => { - const contacts: Partial[] = [{ email: 'test1@example.com' }]; - - (supabaseClient.rpc as jest.Mock).mockReturnValueOnce({ - data: null, - error: null - }); - - const result = await getEnrichmentCache(contacts, EnrichmentService); - - expect(supabaseClient.rpc as jest.Mock).toHaveBeenCalledWith( - 'enriched_most_recent', - { - emails: ['test1@example.com'] - } - ); - expect(result).toEqual([]); - }); - - it('should throw an error when the RPC call fails', async () => { - const contacts: Partial[] = [{ email: 'test1@example.com' }]; - - (supabaseClient.rpc as jest.Mock).mockReturnValueOnce({ - data: null, - error: { message: 'RPC error' } - }); - - await expect( - getEnrichmentCache(contacts, EnrichmentService) - ).rejects.toThrow('RPC error'); - - expect(supabaseClient.rpc as jest.Mock).toHaveBeenCalledWith( - 'enriched_most_recent', - { - emails: ['test1@example.com'] - } - ); - }); - - it('should filter out invalid results after parsing', async () => { - const contacts: Partial[] = [{ email: 'test1@example.com' }]; - - const rpcData = [{ result: { data: [] }, engine: 'testEngine' }]; - - (supabaseClient.rpc as jest.Mock).mockReturnValueOnce({ - data: rpcData, - error: null - }); - - (EnrichmentService.parseResult as jest.Mock).mockReturnValueOnce([]); - - const result = await getEnrichmentCache(contacts, EnrichmentService); - - expect(result).toEqual([]); - }); -}); - -describe('enrichPersonSync', () => { - const enrichmentsDB = mockEnrichmentsDB(); - - beforeEach(() => { - jest.clearAllMocks(); - }); - - it('should enrich all contacts and attach person_id', async () => { - const contacts = [ - { id: 'person-1', email: 'test1@example.com' }, - { id: 'person-2', email: 'test2@example.com' } - ]; - - const enrichmentResults = [ - { data: [{ email: 'test1@example.com' }] }, - { data: [{ email: 'test2@example.com' }] } - ]; - - (EnrichmentService.enrichSync as jest.Mock) - .mockResolvedValueOnce(enrichmentResults[0]) - .mockResolvedValueOnce(enrichmentResults[1]); - - await enrichPersonSync(enrichmentsDB, contacts); - - expect(enrichmentsDB.redactedTask).toHaveBeenCalled(); - expect(enrichmentsDB.enrich).toHaveBeenCalledTimes(2); - // Verify person_id was attached - expect(enrichmentResults[0].data[0]).toHaveProperty( - 'person_id', - 'person-1' - ); - expect(enrichmentResults[1].data[0]).toHaveProperty( - 'person_id', - 'person-2' - ); - }); - - it('should return not enriched contacts', async () => { - const contacts = [ - { id: 'person-1', email: 'test1@example.com' }, - { id: 'person-2', email: 'test2@example.com' } - ]; - - const enrichmentResults = [{ data: [{ email: 'test1@example.com' }] }]; - - (EnrichmentService.enrichSync as jest.Mock) - .mockResolvedValueOnce(enrichmentResults[0]) - .mockResolvedValueOnce(null); - - const result = await enrichPersonSync(enrichmentsDB, contacts); - - expect(enrichmentsDB.redactedTask).toHaveBeenCalled(); - expect(enrichmentsDB.enrich).toHaveBeenCalledTimes(1); - expect(result).toEqual([{ id: 'person-2', email: 'test2@example.com' }]); - }); - - it('should handle no enrichments gracefully', async () => { - const contacts = [ - { id: 'person-1', email: 'test1@example.com' }, - { id: 'person-2', email: 'test2@example.com' } - ]; - - (EnrichmentService.enrichSync as jest.Mock) - .mockResolvedValueOnce(null) - .mockResolvedValueOnce(null); - - const result = await enrichPersonSync(enrichmentsDB, contacts); - - expect(enrichmentsDB.redactedTask).toHaveBeenCalled(); - expect(enrichmentsDB.enrich).not.toHaveBeenCalled(); - expect(result).toEqual(contacts); - }); -}); - -describe('enrichPersonAsync', () => { - const enrichmentsDB = mockEnrichmentsDB(); - - beforeEach(() => { - jest.clearAllMocks(); - }); - - it('should enrich contacts asynchronously', async () => { - const contacts = [{ id: 'person-1', email: 'test1@example.com' }]; - - const task = { id: '1' }; - const webhook = `${ENV.LEADMINER_API_HOST}/api/enrich/webhook/${task.id}`; - const resultData = { data: [{ email: 'test1@example.com' }] }; - - (EnrichmentService.enrichAsync as jest.Mock).mockReturnValue(resultData); - (enrichmentsDB.redactedTask as jest.Mock).mockReturnValue({ id: '1' }); - - await enrichPersonAsync(enrichmentsDB, contacts); - - expect(enrichmentsDB.redactedTask).toHaveBeenCalled(); - expect(EnrichmentService.enrichAsync as jest.Mock).toHaveBeenCalledWith( - contacts, - webhook - ); - // Verify contacts_map was attached - expect( - ( - resultData as { - contacts_map: Array<{ email: string; person_id: string }>; - } - ).contacts_map - ).toEqual([{ email: 'test1@example.com', person_id: 'person-1' }]); - expect(enrichmentsDB.enrich).toHaveBeenCalledWith([resultData]); - }); - - it('should handle no result from async enrichment gracefully', async () => { - const contacts = [{ email: 'test1@example.com' }]; - - const task = { id: '1' }; - const webhook = `${ENV.LEADMINER_API_HOST}/api/enrich/webhook/${task.id}`; - - (EnrichmentService.enrichAsync as jest.Mock).mockReturnValue(null); - - await enrichPersonAsync(enrichmentsDB, contacts); - - expect(enrichmentsDB.redactedTask).toHaveBeenCalled(); - expect(EnrichmentService.enrichAsync as jest.Mock).toHaveBeenCalledWith( - contacts, - webhook - ); - expect(enrichmentsDB.enrich).not.toHaveBeenCalled(); - }); -}); diff --git a/backend/test/unit/enrichment-service/thedig/client.test.ts b/backend/test/unit/enrichment-service/thedig/client.test.ts deleted file mode 100644 index 7a91cdc4e..000000000 --- a/backend/test/unit/enrichment-service/thedig/client.test.ts +++ /dev/null @@ -1,206 +0,0 @@ -import { - afterEach, - beforeEach, - describe, - expect, - it, - jest -} from '@jest/globals'; - -import axios from 'axios'; -import { Logger } from 'winston'; -import ThedigApi, { - EnrichPersonRequest, - EnrichPersonResponse -} from '../../../../src/services/enrichment/thedig/client'; -import { - Distribution, - TokenBucketRateLimiter -} from '../../../../src/services/rate-limiter'; - -// Mock dependencies -jest.mock('axios'); - -jest.mock('ioredis'); - -jest.mock('../../../../src/config', () => ({ - LEADMINER_API_LOG_LEVEL: 'debug' -})); - -const mockAxiosInstance = { - get: jest.fn(), - post: jest.fn() -} as unknown as jest.Mocked; -(axios.create as jest.Mock).mockReturnValue(mockAxiosInstance); - -const THEDIG_THROTTLE_REQUESTS = 5; -const THEDIG_THROTTLE_INTERVAL = 0.1; - -const RATE_LIMITER = new TokenBucketRateLimiter({ - executeEvenly: true, - uniqueKey: 'email_verification_theDig_test', - distribution: Distribution.Memory, - requests: THEDIG_THROTTLE_REQUESTS, - intervalSeconds: THEDIG_THROTTLE_INTERVAL -}); - -describe('ThedigApi', () => { - let mockLogger: jest.Mocked; - let theDigClient: ThedigApi; - - beforeEach(() => { - mockLogger = { error: jest.fn() } as unknown as jest.Mocked; - theDigClient = new ThedigApi( - { - url: 'https://api.example.com', - apiToken: 'test-token', - rateLimiter: RATE_LIMITER - }, - mockLogger - ); - }); - - afterEach(() => { - jest.clearAllMocks(); - }); - - describe('enrich', () => { - it('should return enriched person data on success', async () => { - const personRequest: EnrichPersonRequest = { - name: 'John Doe', - email: 'john@example.com' - }; - const personResponse: EnrichPersonResponse = { - email: 'john@example.com', - name: 'John Doe', - givenName: 'John', - familyName: 'Doe', - jobTitle: ['Engineer'], - worksFor: ['Tech Inc.'], - statusCode: 200 - }; - - mockAxiosInstance.post.mockResolvedValue({ data: personResponse }); - - const result = await theDigClient.enrich(personRequest); - - expect(mockAxiosInstance.post).toHaveBeenCalledWith( - '/person/', - personRequest - ); - expect(result).toEqual(personResponse); - }); - - it('should properly use rate limiter', async () => { - const personRequest: EnrichPersonRequest = { - name: 'John Doe', - email: 'john@example.com' - }; - - // Mock the Axios POST response - mockAxiosInstance.post.mockResolvedValue({ - data: { - email: personRequest.email, - name: personRequest.name - } as EnrichPersonResponse - }); - - const startTime = Date.now(); - const requests = Array.from({ length: THEDIG_THROTTLE_REQUESTS * 2 }).map( - () => theDigClient.enrich(personRequest) - ); - await Promise.all(requests); - const totalTime = Date.now() - startTime; - - expect(totalTime).toBeGreaterThanOrEqual( - THEDIG_THROTTLE_INTERVAL * (requests.length - 1) - ); - }); - - it('should log an error and throw if API request fails', async () => { - const personRequest: EnrichPersonRequest = { - name: 'John Doe', - email: 'john@example.com' - }; - const error = new Error('Network Error'); - mockAxiosInstance.post.mockRejectedValue(error); - - await expect(theDigClient.enrich(personRequest)).rejects.toThrow( - 'Network Error' - ); - expect(mockLogger.error).toHaveBeenCalledWith( - expect.stringContaining('[ThedigApi:enrich]'), - error - ); - }); - }); - - describe('enrichBulk', () => { - it('should return status and token on successful bulk enrichment', async () => { - const persons = [{ email: 'john@example.com' }]; - const webhookUrl = 'https://webhook.example.com'; - const mockResponseToken = 'bulk-token-123'; - - mockAxiosInstance.post.mockResolvedValue({ - data: mockResponseToken - }); - - const result = await theDigClient.enrichBulk(persons, webhookUrl); - - expect(mockAxiosInstance.post).toHaveBeenCalledWith( - `/person/bulk?endpoint=${webhookUrl}`, - persons - ); - expect(result).toEqual({ - status: 'running', - success: true, - token: mockResponseToken - }); - }); - - it('should properly use rate limiter', async () => { - const personRequest = [ - { - name: 'John Doe', - email: 'john@example.com' - }, - { - name: 'John Doe 1', - email: 'john@example.com' - } - ]; - - // Mock the Axios POST response - mockAxiosInstance.post.mockResolvedValue({ - data: 'bulk-token-123' - }); - - const startTime = Date.now(); - - const requests = Array.from({ length: THEDIG_THROTTLE_REQUESTS * 2 }).map( - () => theDigClient.enrichBulk(personRequest, 'webhook') - ); - await Promise.all(requests); - const totalTime = Date.now() - startTime; - - expect(totalTime).toBeGreaterThanOrEqual( - THEDIG_THROTTLE_INTERVAL * (requests.length - 1) - ); - }); - - it('should log an error and throw if bulk request fails', async () => { - const persons = [{ email: 'john@example.com' }]; - const webhookUrl = 'https://webhook.example.com'; - const error = new Error('Bulk request error'); - mockAxiosInstance.post.mockRejectedValue(error); - - await expect( - theDigClient.enrichBulk(persons, webhookUrl) - ).rejects.toThrow('Bulk request error'); - expect(mockLogger.error).toHaveBeenCalledWith( - expect.stringContaining('[ThedigApi:enrichBulk]'), - error - ); - }); - }); -}); diff --git a/backend/test/unit/enrichment-service/thedig/enricher.test.ts b/backend/test/unit/enrichment-service/thedig/enricher.test.ts deleted file mode 100644 index a8e74427c..000000000 --- a/backend/test/unit/enrichment-service/thedig/enricher.test.ts +++ /dev/null @@ -1,295 +0,0 @@ -import { beforeEach, describe, expect, it, jest } from '@jest/globals'; -import { Logger } from 'winston'; -import ThedigApi, { - EnrichPersonResponse -} from '../../../../src/services/enrichment/thedig/client'; - -import { EngineResponse } from '../../../../src/services/enrichment/Engine'; -import { Person } from '../../../../src/db/types'; -import Thedig from '../../../../src/services/enrichment/thedig'; - -describe('Thedig', () => { - let mockClient: jest.Mocked; - let mockLogger: jest.Mocked; - let enricher: Thedig; - - const enrichmentResponseMock: EnrichPersonResponse = { - email: 'test@example.com', - name: 'Jane Doe', - givenName: 'Jane', - familyName: 'Doe', - jobTitle: ['Software Engineer'], - worksFor: ['Tech Co.'], - image: ['https://example.com/pic.jpg'], - homeLocation: 'City, State', - workLocation: 'City, State', - alternateName: ['JDoe'], - sameAs: ['https://linkedin.com/in/janedoe'], - identifier: ['jane.doe'], - statusCode: 200 - }; - const enrichmentMappedMock: EngineResponse = { - engine: 'thedig', - data: [ - { - email: 'test@example.com', - name: 'Jane Doe', - givenName: 'Jane', - familyName: 'Doe', - jobTitle: 'Software Engineer', - alternateName: ['JDoe'], - organization: 'Tech Co.', - image: 'https://example.com/pic.jpg', - identifiers: ['jane.doe'], - location: 'City, State, City, State', - sameAs: ['https://linkedin.com/in/janedoe'] - } - ], - raw_data: [enrichmentResponseMock] - }; - - beforeEach(() => { - mockClient = { - enrich: jest.fn(), - enrichBulk: jest.fn() - } as Partial> as jest.Mocked; - - mockLogger = { - debug: jest.fn(), - info: jest.fn(), - warn: jest.fn(), - error: jest.fn() - } as unknown as jest.Mocked; - - enricher = new Thedig(mockClient, mockLogger); - }); - - describe('parseResult', () => { - it('should map EnrichPersonResponse data correctly', () => { - const result = enricher.parseResult([enrichmentResponseMock]); - expect(result).toEqual(enrichmentMappedMock); - }); - - it('should map partial EnrichpersonResponse data correctly', () => { - const enrichmentResultMock = [ - { - email: 'valid@example.com', - name: 'Valid User', - givenName: 'Valid', - familyName: 'User', - jobTitle: ['Software Engineer'], - worksFor: ['Leadminer'], - image: [ - 'https://x.com/validuser/profile/pic.png', - 'https://linkedin.com/in/validuser/pic.png' - ], - homeLocation: 'country, city', - workLocation: 'country, city', - alternateName: ['valid_user_123'], - sameAs: [ - 'https://linkedin.com/in/validuser', - 'https://x.com/validuser/' - ], - identifier: ['valid.user'], - statusCode: 200 - }, - { - email: 'partial@example.com', - name: 'Partial User', - givenName: '', - familyName: 'User', - worksFor: ['Leadminer'], - workLocation: 'country, city', - sameAs: [ - 'https://linkedin.com/in/partialuser', - 'https://x.com/partialuser/' - ], - statusCode: 200 - }, - { - email: 'partial2@example.com', - name: 'Partial2 User', - givenName: 'Partial2', - familyName: 'User', - worksFor: ['Leadminer'], - statusCode: 200 - } - ]; - const result = enricher.parseResult(enrichmentResultMock); - expect(result).toEqual({ - engine: 'thedig', - data: [ - { - email: 'valid@example.com', - name: 'Valid User', - givenName: 'Valid', - familyName: 'User', - jobTitle: 'Software Engineer', - organization: 'Leadminer', - identifiers: ['valid.user'], - location: 'country, city, country, city', - sameAs: [ - 'https://linkedin.com/in/validuser', - 'https://x.com/validuser/' - ], - alternateName: ['valid_user_123'], - image: 'https://x.com/validuser/profile/pic.png' - }, - { - email: 'partial@example.com', - name: 'Partial User', - givenName: undefined, - familyName: 'User', - jobTitle: undefined, - organization: 'Leadminer', - identifiers: undefined, - location: 'country, city', - sameAs: [ - 'https://linkedin.com/in/partialuser', - 'https://x.com/partialuser/' - ], - alternateName: undefined, - image: undefined - }, - { - email: 'partial2@example.com', - name: 'Partial2 User', - givenName: 'Partial2', - familyName: 'User', - organization: 'Leadminer' - } - ], - raw_data: enrichmentResultMock - }); - }); - }); - - describe('enrich', () => { - it('should call enrich and map the response correctly', async () => { - const person: Partial = { - name: 'John doe', - email: 'test@example.com' - }; - - mockClient.enrich.mockResolvedValue(enrichmentResponseMock); - - const result = await enricher.enrichSync(person); - - expect(mockClient.enrich).toHaveBeenCalledWith(person); - expect(result).toEqual(enrichmentMappedMock); - }); - - it('should not enriched if statusCode=203 and sameAs is empty', async () => { - const person: Partial = { - name: 'John doe', - email: 'test@example.com' - }; - - const response = { - ...enrichmentResponseMock, - sameAs: [], - statusCode: 203 - }; - mockClient.enrich.mockResolvedValue(response); - - const result = await enricher.enrichSync(person); - - expect(mockClient.enrich).toHaveBeenCalledWith(person); - expect(result).toEqual({ - engine: 'thedig', - data: [], - raw_data: [response] - }); - }); - - it('should not enriched if statusCode 203 sameAs is not empty', async () => { - const person: Partial = { - name: 'John doe', - email: 'test@example.com' - }; - - const response = { - ...enrichmentResponseMock, - statusCode: 203 - }; - mockClient.enrich.mockResolvedValue(response); - - const result = await enricher.enrichSync(person); - - expect(mockClient.enrich).toHaveBeenCalledWith(person); - expect(result).toEqual({ - ...enrichmentMappedMock, - raw_data: [response] - }); - }); - - it('should throw an error if enrich fails', async () => { - const person: Partial = { email: 'test@example.com' }; - const error = new Error('Network error'); - mockClient.enrich.mockRejectedValue(error); - - await expect(enricher.enrichSync(person)).rejects.toThrow( - 'Network error' - ); - }); - }); - - describe('enrichAsync', () => { - it('should call enrichBulk with correct parameters and handle a successful response', async () => { - const webhook = 'https://webhook-url.com'; - const persons: Partial[] = [ - { email: 'john@example.com', name: 'John Doe' }, - { email: 'jane@example.com', name: 'Jane Doe' } - ]; - const mockResponse = { - success: true, - status: 'ok', - token: 'test-token' - }; - - mockClient.enrichBulk.mockResolvedValue(mockResponse); - - const result = await enricher.enrichAsync(persons, webhook); - - expect(mockClient.enrichBulk).toHaveBeenCalledWith(persons, webhook); - expect(result).toEqual({ - data: [], - raw_data: [], - engine: 'thedig', - token: mockResponse.token - }); - }); - - it('should throw an error if enrichBulk fails', async () => { - const persons: Partial[] = [ - { email: 'test@example.com', name: 'John Doe' } - ]; - const webhook = 'https://webhook-url.com'; - const error = new Error('Network error'); - - mockClient.enrichBulk.mockRejectedValue(error); - - await expect(enricher.enrichAsync(persons, webhook)).rejects.toThrow( - 'Network error' - ); - }); - - it('should throw an error if response is unsuccessful', async () => { - const persons: Partial[] = [ - { email: 'test@example.com', name: 'John Doe' } - ]; - const webhook = 'https://webhook-url.com'; - const mockResponse = { - success: false, - status: 'error', - token: 'error-token' - }; - - mockClient.enrichBulk.mockResolvedValue(mockResponse); - - await expect(enricher.enrichAsync(persons, webhook)).rejects.toThrow( - 'Failed to upload emails to enrichment.' - ); - }); - }); -}); diff --git a/backend/test/unit/enrichment-service/voilanorbert/client.test.ts b/backend/test/unit/enrichment-service/voilanorbert/client.test.ts deleted file mode 100644 index b414f6ec5..000000000 --- a/backend/test/unit/enrichment-service/voilanorbert/client.test.ts +++ /dev/null @@ -1,135 +0,0 @@ -import { - afterEach, - beforeEach, - describe, - expect, - it, - jest -} from '@jest/globals'; - -import axios from 'axios'; -import { Logger } from 'winston'; -import VoilanorbertApi from '../../../../src/services/enrichment/voilanorbert/client'; -import { logError } from '../../../../src/utils/axios'; -import { - Distribution, - TokenBucketRateLimiter -} from '../../../../src/services/rate-limiter'; - -jest.mock('axios'); - -jest.mock('ioredis'); - -jest.mock('../../../../src/config', () => ({ - LEADMINER_API_LOG_LEVEL: 'debug' -})); - -const mockAxiosInstance = { - get: jest.fn(), - post: jest.fn() -}; -(axios.create as jest.Mock).mockReturnValue(mockAxiosInstance); - -jest.mock('../../../../src/utils/axios', () => ({ - logError: jest.fn() -})); - -const VOILANORBERT_THROTTLE_REQUESTS = 5; -const VOILANORBERT_THROTTLE_INTERVAL = 0.1; - -const RATE_LIMITER = new TokenBucketRateLimiter({ - executeEvenly: true, - uniqueKey: 'email_verification_voilanorbert_test', - distribution: Distribution.Memory, - requests: VOILANORBERT_THROTTLE_REQUESTS, - intervalSeconds: VOILANORBERT_THROTTLE_INTERVAL -}); - -describe('VoilanorbertApi', () => { - const mockLogger = { - info: jest.fn(), - error: jest.fn() - } as unknown as Logger; - const config = { - url: 'https://api.voilanorbert.com', - username: 'testUser', - apiToken: 'testToken', - rateLimiter: RATE_LIMITER - }; - - let voilanorbert: VoilanorbertApi; - - beforeEach(() => { - voilanorbert = new VoilanorbertApi(config, mockLogger); - }); - - afterEach(() => { - jest.clearAllMocks(); - }); - - describe('enrich', () => { - it('should successfully enrich emails', async () => { - const emails = ['test@example.com']; - const webhook = 'webhook-url'; - - const mockResponse = { - status: 'success', - success: true, - token: 'testToken' - }; - - mockAxiosInstance.post.mockReturnValue({ data: mockResponse }); - - const response = await voilanorbert.enrich(emails, webhook); - - expect(response).toEqual(mockResponse); - expect(axios.create).toHaveBeenCalledWith({ - baseURL: config.url, - headers: {}, - auth: { - username: config.username, - password: config.apiToken - } - }); - }); - - it('should properly use rate limiter', async () => { - const emails = [ - 'test1@example.com', - 'test2@example.com', - 'test3@example.com' - ]; - - const startTime = Date.now(); - const requests = Array.from({ - length: VOILANORBERT_THROTTLE_REQUESTS * 2 - }).map(() => voilanorbert.enrich(emails, 'webhook-url')); - await Promise.all(requests); - - const totalTime = Date.now() - startTime; - - expect(totalTime).toBeGreaterThanOrEqual( - VOILANORBERT_THROTTLE_INTERVAL * (requests.length - 1) - ); - }); - - it('should log an error and throw it when the request fails', async () => { - const emails = ['test@example.com']; - const webhook = 'webhook-url'; - const mockError = new Error('Request failed'); - - mockAxiosInstance.post.mockImplementation(() => - Promise.reject(mockError) - ); - - await expect(voilanorbert.enrich(emails, webhook)).rejects.toThrow( - mockError - ); - expect(logError).toHaveBeenCalledWith( - mockError, - expect.any(String), - mockLogger - ); - }); - }); -}); diff --git a/backend/test/unit/enrichment-service/voilanorbert/enricher.test.ts b/backend/test/unit/enrichment-service/voilanorbert/enricher.test.ts deleted file mode 100644 index ed91d9e4c..000000000 --- a/backend/test/unit/enrichment-service/voilanorbert/enricher.test.ts +++ /dev/null @@ -1,188 +0,0 @@ -import { - afterEach, - beforeEach, - describe, - expect, - it, - jest -} from '@jest/globals'; - -import { Logger } from 'winston'; -import VoilanorbertApi, { - ResponseWebhook -} from '../../../../src/services/enrichment/voilanorbert/client'; -import { Person } from '../../../../src/db/types'; -import Voilanorbert from '../../../../src/services/enrichment/voilanorbert'; - -jest.mock('../../../../src/services/enrichment/voilanorbert/client'); - -describe('Voilanorbert', () => { - let enricher: Voilanorbert; - const mockLogger = { - debug: jest.fn(), - error: jest.fn() - } as unknown as Logger; - const mockClient = { - enrich: jest.fn(), - enrichBulk: jest.fn() - } as Partial> as jest.Mocked; - - beforeEach(() => { - enricher = new Voilanorbert(mockClient, mockLogger); - }); - - afterEach(() => { - jest.clearAllMocks(); - }); - - describe('enrichAsync', () => { - it('should successfully call client.enrich and handle the response', async () => { - const persons: Partial[] = [ - { email: 'test1@example.com' }, - { email: 'test2@example.com' } - ]; - const webhook = 'test-webhook'; - const mockResponse = { - success: true, - token: 'testToken', - status: 'ok' - }; - - mockClient.enrich.mockResolvedValue(mockResponse); - - const result = await enricher.enrichAsync(persons, webhook); - - expect(mockClient.enrich).toHaveBeenCalledWith( - ['test1@example.com', 'test2@example.com'], - webhook - ); - expect(result).toEqual({ - engine: 'voilanorbert', - data: [], - raw_data: [], - token: mockResponse.token - }); - expect(mockLogger.debug).toHaveBeenCalledWith( - 'Got Voilanorbert.enrichAsync request', - persons - ); - }); - - it('should throw an error if client.enrich fails', async () => { - const persons: Partial[] = [{ email: 'test@example.com' }]; - const webhook = 'test-webhook'; - const errorMessage = 'Failed to upload emails to enrichment.'; - mockClient.enrich.mockResolvedValue({ - token: '', - status: 'error', - success: false - }); - - await expect(enricher.enrichAsync(persons, webhook)).rejects.toThrow( - errorMessage - ); - - expect(mockLogger.debug).toHaveBeenCalledWith( - 'Got Voilanorbert.enrichAsync request', - persons - ); - }); - }); - - describe('parseResult', () => { - it('should correctly map ResponseWebhook to EnricherResult format', () => { - const mockWebhookResult: ResponseWebhook = { - id: '123', - token: 'testToken', - results: [ - { - email: 'valid@example.com', - fullName: 'Valid User', - organization: 'Leadminer', - title: 'Senior Developer', - location: 'country, city', - twitter: 'twitter.com/validUser', - linkedin: 'linkedin.com/in/validUser', - facebook: 'facebook.com/validUser' - }, - { - email: 'partial@example.com', - fullName: 'Partial User', - organization: 'Leadminer', - title: 'Intern', - location: 'country, city', - twitter: '', - linkedin: 'linkedin.com/in/partialUser', - facebook: 'facebook.com/partialUser' - }, - { - email: 'partial2@example.com', - fullName: 'Partial User 2', - organization: '', - title: '', - location: '', - twitter: '', - linkedin: '', - facebook: '' - }, - { - email: 'empty@example.com', - fullName: '', - organization: '', - title: '', - location: '', - twitter: '', - linkedin: '', - facebook: '' - } - ] - }; - - const expectedMappedData = { - engine: 'voilanorbert', - data: [ - { - email: 'valid@example.com', - name: 'Valid User', - location: 'country, city', - organization: 'Leadminer', - jobTitle: 'Senior Developer', - sameAs: [ - 'facebook.com/validUser', - 'linkedin.com/in/validUser', - 'twitter.com/validUser' - ], - image: undefined - }, - { - email: 'partial@example.com', - name: 'Partial User', - location: 'country, city', - organization: 'Leadminer', - jobTitle: 'Intern', - sameAs: ['facebook.com/partialUser', 'linkedin.com/in/partialUser'], - image: undefined - }, - { - email: 'partial2@example.com', - name: 'Partial User 2', - image: undefined, - jobTitle: undefined, - location: undefined, - organization: undefined, - sameAs: undefined - } - ], - raw_data: mockWebhookResult.results - }; - - const result = enricher.parseResult([mockWebhookResult]); - - expect(result).toEqual(expectedMappedData); - expect(mockLogger.debug).toHaveBeenCalledWith( - '[Voilanorbert]-[parseResult]: Parsing enrichment results', - [mockWebhookResult] - ); - }); - }); -}); diff --git a/frontend/src/components/mining/buttons/ExportContacts.vue b/frontend/src/components/mining/buttons/ExportContacts.vue index 440d1e58a..517fadff0 100644 --- a/frontend/src/components/mining/buttons/ExportContacts.vue +++ b/frontend/src/components/mining/buttons/ExportContacts.vue @@ -1,14 +1,8 @@