diff --git a/backend/src/db/environment.json b/backend/src/db/environment.json index 7b4d508..1d2e8d3 100644 --- a/backend/src/db/environment.json +++ b/backend/src/db/environment.json @@ -1,8 +1,5 @@ { "KEYWORDS": [ - "Java", - "Spring", - "RabbitMQ", - "Docker" + "Java" ] } \ No newline at end of file diff --git a/backend/src/pipeline/scrapeAllSources.js b/backend/src/pipeline/scrapeAllSources.js index f2705c4..5296814 100644 --- a/backend/src/pipeline/scrapeAllSources.js +++ b/backend/src/pipeline/scrapeAllSources.js @@ -10,21 +10,88 @@ function normalizeJob(job, keyword, adapter) { }; } -function mergeKeywords(existing, incoming) { - const keywords = new Set([ - ...(existing.keywords || []), - ...(existing.keyword ? [existing.keyword] : []), - ...(existing.palavraChave ? [existing.palavraChave] : []), - ...(incoming.keywords || []), - ...(incoming.keyword ? [incoming.keyword] : []), - ...(incoming.palavraChave ? [incoming.palavraChave] : []), - ]); +function normalizeComparableText(value) { + return String(value || "") + .normalize("NFD") + .replace(/\p{Diacritic}/gu, "") + .toLowerCase() + .replace(/[^\p{L}\p{N}]+/gu, " ") + .trim() + .replace(/\s+/g, " "); +} + +function normalizeComparableUrl(value) { + const rawValue = String(value || "").trim(); + + if (!rawValue) { + return ""; + } + + try { + const parsedUrl = new URL(rawValue); + parsedUrl.search = ""; + parsedUrl.hash = ""; + return `${parsedUrl.origin}${parsedUrl.pathname}`.replace(/\/+$/, ""); + } catch { + return rawValue.split(/[?#]/)[0].replace(/\/+$/, ""); + } +} + +function getMergedKeywords(...jobs) { + return [ + ...new Set( + jobs.flatMap((job) => + [ + ...(Array.isArray(job.keywords) ? job.keywords : []), + job.keyword, + job.palavraChave, + job.palavra, + ] + .filter(Boolean) + .map((keyword) => String(keyword).trim()), + ), + ), + ]; +} - const mergedKeywords = [...keywords].filter(Boolean); +function getMergedSources(...jobs) { + return [ + ...new Set( + jobs.flatMap((job) => + [ + ...(Array.isArray(job.sources) ? job.sources : []), + job.source, + ] + .filter(Boolean) + .map((source) => String(source).trim()), + ), + ), + ]; +} + +function pickPreferredValue(...values) { + return ( + values + .map((value) => String(value || "").trim()) + .filter(Boolean) + .sort((left, right) => right.length - left.length)[0] || "" + ); +} + +function mergeKeywords(existing, incoming) { + const mergedKeywords = getMergedKeywords(existing, incoming); + const mergedSources = getMergedSources(existing, incoming); return { ...existing, ...incoming, + titulo: pickPreferredValue(existing.titulo, incoming.titulo, existing.title, incoming.title), + empresa: pickPreferredValue(existing.empresa, incoming.empresa, existing.company, incoming.company), + local: pickPreferredValue(existing.local, incoming.local, existing.location, incoming.location), + link: pickPreferredValue(existing.link, incoming.link, existing.jobUrl, incoming.jobUrl), + jobUrl: pickPreferredValue(existing.jobUrl, incoming.jobUrl, existing.link, incoming.link), + source: mergedSources.join(", ") || existing.source || incoming.source || "", + sources: mergedSources, keyword: mergedKeywords[0] || "", palavraChave: mergedKeywords[0] || "", keywords: mergedKeywords, @@ -32,16 +99,33 @@ function mergeKeywords(existing, incoming) { }; } +function buildDedupKey(job) { + const title = normalizeComparableText(job.titulo || job.title); + const company = normalizeComparableText(job.empresa || job.company); + const location = normalizeComparableText(job.local || job.location); + + if (title && company && location) { + return `identity:${title}|${company}|${location}`; + } + + if (title && company) { + return `identity:${title}|${company}|${location || "sem-local"}`; + } + + const normalizedLink = normalizeComparableUrl(job.link || job.jobUrl); + + if (normalizedLink) { + return `url:${normalizedLink}`; + } + + return `fallback:${title}|${company}|${location}|${normalizeComparableText(job.source)}`; +} + function dedupeJobs(jobs) { const unique = new Map(); for (const job of jobs) { - const key = - job.link || - job.jobUrl || - `${job.source}-${job.titulo || job.title}-${job.empresa || job.company}-${job.local || job.location}`; - - if (!key) continue; + const key = buildDedupKey(job); if (unique.has(key)) { const existing = unique.get(key); @@ -49,16 +133,15 @@ function dedupeJobs(jobs) { continue; } + const mergedKeywords = getMergedKeywords(job); + const mergedSources = getMergedSources(job); + unique.set(key, { ...job, + source: mergedSources.join(", ") || job.source || "", + sources: mergedSources, palavra: job.keyword || job.palavraChave || job.palavra || "", - keywords: [ - ...new Set( - [job.keyword, job.palavraChave, ...(job.keywords || [])].filter( - Boolean, - ), - ), - ], + keywords: mergedKeywords, }); } diff --git a/backend/tests/unit/services/pipeline/scrapeAllSources.test.js b/backend/tests/unit/services/pipeline/scrapeAllSources.test.js index 195407c..4b38ece 100644 --- a/backend/tests/unit/services/pipeline/scrapeAllSources.test.js +++ b/backend/tests/unit/services/pipeline/scrapeAllSources.test.js @@ -160,6 +160,41 @@ describe("scrapeAllSources", () => { expect(jobs[0].keywords).toEqual(["React", "Node"]); }); + it("deduplica a mesma vaga entre plataformas pelo título, empresa e local", async () => { + const adapters = [ + { + sourceName: "LinkedIn", + search: vi.fn().mockResolvedValue([ + { + titulo: "Desenvolvedor(a) Java Jr", + empresa: "Sankhya", + local: "Brasil", + link: "https://www.linkedin.com/jobs/view/desenvolvedor-java-jr-123?refId=abc&trackingId=123", + }, + ]), + }, + { + sourceName: "Greenhouse", + search: vi.fn().mockResolvedValue([ + { + titulo: "desenvolvedor a java jr", + empresa: "Sankhya", + local: "Brasil", + link: "https://boards.greenhouse.io/sankhya/jobs/123", + }, + ]), + }, + ]; + + const jobs = await scrapeAllSources(adapters, { keywords: ["Java"] }); + + expect(jobs).toHaveLength(1); + expect(jobs[0]).toMatchObject({ + empresa: "Sankhya", + local: "Brasil", + }); + }); + it("ignora retorno inválido do adapter e chama logWarn", async () => { const adapters = [ { diff --git a/frontend/src/hooks/useJobsFiltering.ts b/frontend/src/hooks/useJobsFiltering.ts index a54460c..a1275e9 100644 --- a/frontend/src/hooks/useJobsFiltering.ts +++ b/frontend/src/hooks/useJobsFiltering.ts @@ -1,21 +1,135 @@ import type { Job } from "@/types/jobs"; import { useMemo, useState } from "react"; +function normalizeComparableText(value?: string | null) { + return String(value || "") + .normalize("NFD") + .replace(/\p{Diacritic}/gu, "") + .toLowerCase() + .replace(/[^\p{L}\p{N}]+/gu, " ") + .trim() + .replace(/\s+/g, " "); +} + +function normalizeComparableLink(value?: string | null) { + const rawValue = String(value || "").trim(); + + if (!rawValue) { + return ""; + } + + try { + const parsedUrl = new URL(rawValue); + parsedUrl.search = ""; + parsedUrl.hash = ""; + return `${parsedUrl.origin}${parsedUrl.pathname}`.replace(/\/+$/, ""); + } catch { + return rawValue.split(/[?#]/)[0].replace(/\/+$/, ""); + } +} + +function splitJobKeywords(job: Job) { + return [ + ...new Set( + [ + ...(Array.isArray(job.keywords) ? job.keywords : []), + ...String(job.palavra || "") + .split(/[,;|]+/) + .map((keyword) => keyword.trim()), + ].filter(Boolean), + ), + ]; +} + +function pickPreferredValue(...values: Array) { + return ( + values + .map((value) => String(value || "").trim()) + .filter(Boolean) + .sort((left, right) => right.length - left.length)[0] || "" + ); +} + +function buildDedupKey(job: Job) { + const title = normalizeComparableText(job.titulo); + const company = normalizeComparableText(job.empresa); + const location = normalizeComparableText(job.local); + + if (title && company && location) { + return `identity:${title}|${company}|${location}`; + } + + if (title && company) { + return `identity:${title}|${company}|${location || "sem-local"}`; + } + + const link = normalizeComparableLink(job.link); + + if (link) { + return `url:${link}`; + } + + return `fallback:${title}|${company}|${location}|${normalizeComparableText(job.source)}`; +} + +function dedupeJobs(jobs: Job[]) { + const unique = new Map(); + + for (const job of jobs) { + const key = buildDedupKey(job); + const existing = unique.get(key); + + if (!existing) { + unique.set(key, { + ...job, + palavra: splitJobKeywords(job).join(", "), + }); + continue; + } + + const mergedKeywords = [...new Set([...splitJobKeywords(existing), ...splitJobKeywords(job)])]; + const mergedSources = [ + ...new Set( + [...(existing.sources ?? []), ...(job.sources ?? []), existing.source, job.source] + .map((source) => String(source || "").trim()) + .filter(Boolean), + ), + ]; + + unique.set(key, { + ...existing, + ...job, + titulo: pickPreferredValue(existing.titulo, job.titulo), + empresa: pickPreferredValue(existing.empresa, job.empresa), + local: pickPreferredValue(existing.local, job.local), + link: pickPreferredValue(existing.link, job.link), + source: mergedSources.join(", ") || existing.source || job.source || "", + palavra: mergedKeywords.join(", "), + keywords: mergedKeywords, + sources: mergedSources.length > 0 ? mergedSources : undefined, + }); + } + + return [...unique.values()]; +} + export function useJobsFiltering(jobs: Job[]) { const [search, setSearch] = useState(""); const [keywordFilter, setKeywordFilter] = useState([]); + const dedupedJobs = useMemo(() => dedupeJobs(jobs), [jobs]); + const keywords = useMemo(() => { - const values = Array.from(new Set(jobs.map((job) => String(job.palavra || "").trim()).filter(Boolean))); + const values = Array.from(new Set(dedupedJobs.flatMap((job) => splitJobKeywords(job)))); return values.sort((a, b) => a.localeCompare(b)); - }, [jobs]); + }, [dedupedJobs]); const filteredJobs = useMemo(() => { - const term = search.trim().toLowerCase(); + const term = normalizeComparableText(search); - return jobs.filter((job) => { - const currentKeyword = String(job.palavra || "").trim(); - const byKeyword = keywordFilter.length === 0 || keywordFilter.includes(currentKeyword); + return dedupedJobs.filter((job) => { + const currentKeywords = splitJobKeywords(job); + const byKeyword = keywordFilter.length === 0 || keywordFilter.some((keyword) => currentKeywords.includes(keyword)); if (!byKeyword) { return false; } @@ -24,13 +138,13 @@ export function useJobsFiltering(jobs: Job[]) { return true; } - const text = [job.titulo, job.empresa, job.local, job.link, job.palavra] - .map((value) => String(value || "").toLowerCase()) - .join(" "); + const text = normalizeComparableText( + [job.titulo, job.empresa, job.local, job.link, job.palavra, ...(job.keywords || [])].join(" "), + ); return text.includes(term); }); - }, [jobs, search, keywordFilter]); + }, [dedupedJobs, search, keywordFilter]); return { search, diff --git a/frontend/src/types/jobs.ts b/frontend/src/types/jobs.ts index 66ac6df..742c4a7 100644 --- a/frontend/src/types/jobs.ts +++ b/frontend/src/types/jobs.ts @@ -1,8 +1,10 @@ export interface Job { palavra?: string | null; + keywords?: string[] | null; titulo?: string | null; empresa?: string | null; source?: string | null; + sources?: string[] | null; local?: string | null; link?: string | null; } diff --git a/frontend/tests/unit/utils/useJobsFiltering.test.ts b/frontend/tests/unit/utils/useJobsFiltering.test.ts index 3813d8c..01de9e9 100644 --- a/frontend/tests/unit/utils/useJobsFiltering.test.ts +++ b/frontend/tests/unit/utils/useJobsFiltering.test.ts @@ -38,4 +38,29 @@ describe("useJobsFiltering", () => { expect(result.current.filteredJobs).toHaveLength(1); expect(result.current.filteredJobs[0].titulo).toBe("Frontend"); }); + + it("deduplica vagas equivalentes e combina palavras-chave", () => { + const duplicatedJobs = [ + { + palavra: "Java", + titulo: "Desenvolvedor(a) Java Jr", + empresa: "Sankhya", + local: "Brasil", + link: "https://www.linkedin.com/jobs/view/desenvolvedor-java-jr-123?refId=abc", + }, + { + palavra: "Spring", + titulo: "desenvolvedor a java jr", + empresa: "Sankhya", + local: "Brasil", + link: "https://boards.greenhouse.io/sankhya/jobs/123", + }, + ]; + + const { result } = renderHook(() => useJobsFiltering(duplicatedJobs)); + + expect(result.current.filteredJobs).toHaveLength(1); + expect(result.current.filteredJobs[0].palavra).toContain("Java"); + expect(result.current.filteredJobs[0].palavra).toContain("Spring"); + }); });