Skip to content

Commit cdc66de

Browse files
BenevanioBenevanio
authored
Fix/job duplication (#64)
2 parents 2b14235 + 8616698 commit cdc66de

6 files changed

Lines changed: 293 additions & 37 deletions

File tree

‎backend/src/db/environment.json‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,5 @@
11
{
22
"KEYWORDS": [
3-
"Java",
4-
"Spring",
5-
"RabbitMQ",
6-
"Docker"
3+
"Java"
74
]
85
}

‎backend/src/pipeline/scrapeAllSources.js‎

Lines changed: 106 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -10,55 +10,138 @@ function normalizeJob(job, keyword, adapter) {
1010
};
1111
}
1212

13-
function mergeKeywords(existing, incoming) {
14-
const keywords = new Set([
15-
...(existing.keywords || []),
16-
...(existing.keyword ? [existing.keyword] : []),
17-
...(existing.palavraChave ? [existing.palavraChave] : []),
18-
...(incoming.keywords || []),
19-
...(incoming.keyword ? [incoming.keyword] : []),
20-
...(incoming.palavraChave ? [incoming.palavraChave] : []),
21-
]);
13+
function normalizeComparableText(value) {
14+
return String(value || "")
15+
.normalize("NFD")
16+
.replace(/\p{Diacritic}/gu, "")
17+
.toLowerCase()
18+
.replace(/[^\p{L}\p{N}]+/gu, " ")
19+
.trim()
20+
.replace(/\s+/g, " ");
21+
}
22+
23+
function normalizeComparableUrl(value) {
24+
const rawValue = String(value || "").trim();
25+
26+
if (!rawValue) {
27+
return "";
28+
}
29+
30+
try {
31+
const parsedUrl = new URL(rawValue);
32+
parsedUrl.search = "";
33+
parsedUrl.hash = "";
34+
return `${parsedUrl.origin}${parsedUrl.pathname}`.replace(/\/+$/, "");
35+
} catch {
36+
return rawValue.split(/[?#]/)[0].replace(/\/+$/, "");
37+
}
38+
}
39+
40+
function getMergedKeywords(...jobs) {
41+
return [
42+
...new Set(
43+
jobs.flatMap((job) =>
44+
[
45+
...(Array.isArray(job.keywords) ? job.keywords : []),
46+
job.keyword,
47+
job.palavraChave,
48+
job.palavra,
49+
]
50+
.filter(Boolean)
51+
.map((keyword) => String(keyword).trim()),
52+
),
53+
),
54+
];
55+
}
2256

23-
const mergedKeywords = [...keywords].filter(Boolean);
57+
function getMergedSources(...jobs) {
58+
return [
59+
...new Set(
60+
jobs.flatMap((job) =>
61+
[
62+
...(Array.isArray(job.sources) ? job.sources : []),
63+
job.source,
64+
]
65+
.filter(Boolean)
66+
.map((source) => String(source).trim()),
67+
),
68+
),
69+
];
70+
}
71+
72+
function pickPreferredValue(...values) {
73+
return (
74+
values
75+
.map((value) => String(value || "").trim())
76+
.filter(Boolean)
77+
.sort((left, right) => right.length - left.length)[0] || ""
78+
);
79+
}
80+
81+
function mergeKeywords(existing, incoming) {
82+
const mergedKeywords = getMergedKeywords(existing, incoming);
83+
const mergedSources = getMergedSources(existing, incoming);
2484

2585
return {
2686
...existing,
2787
...incoming,
88+
titulo: pickPreferredValue(existing.titulo, incoming.titulo, existing.title, incoming.title),
89+
empresa: pickPreferredValue(existing.empresa, incoming.empresa, existing.company, incoming.company),
90+
local: pickPreferredValue(existing.local, incoming.local, existing.location, incoming.location),
91+
link: pickPreferredValue(existing.link, incoming.link, existing.jobUrl, incoming.jobUrl),
92+
jobUrl: pickPreferredValue(existing.jobUrl, incoming.jobUrl, existing.link, incoming.link),
93+
source: mergedSources.join(", ") || existing.source || incoming.source || "",
94+
sources: mergedSources,
2895
keyword: mergedKeywords[0] || "",
2996
palavraChave: mergedKeywords[0] || "",
3097
keywords: mergedKeywords,
3198
palavra: mergedKeywords[0] || "",
3299
};
33100
}
34101

102+
function buildDedupKey(job) {
103+
const title = normalizeComparableText(job.titulo || job.title);
104+
const company = normalizeComparableText(job.empresa || job.company);
105+
const location = normalizeComparableText(job.local || job.location);
106+
107+
if (title && company && location) {
108+
return `identity:${title}|${company}|${location}`;
109+
}
110+
111+
if (title && company) {
112+
return `identity:${title}|${company}|${location || "sem-local"}`;
113+
}
114+
115+
const normalizedLink = normalizeComparableUrl(job.link || job.jobUrl);
116+
117+
if (normalizedLink) {
118+
return `url:${normalizedLink}`;
119+
}
120+
121+
return `fallback:${title}|${company}|${location}|${normalizeComparableText(job.source)}`;
122+
}
123+
35124
function dedupeJobs(jobs) {
36125
const unique = new Map();
37126

38127
for (const job of jobs) {
39-
const key =
40-
job.link ||
41-
job.jobUrl ||
42-
`${job.source}-${job.titulo || job.title}-${job.empresa || job.company}-${job.local || job.location}`;
43-
44-
if (!key) continue;
128+
const key = buildDedupKey(job);
45129

46130
if (unique.has(key)) {
47131
const existing = unique.get(key);
48132
unique.set(key, mergeKeywords(existing, job));
49133
continue;
50134
}
51135

136+
const mergedKeywords = getMergedKeywords(job);
137+
const mergedSources = getMergedSources(job);
138+
52139
unique.set(key, {
53140
...job,
141+
source: mergedSources.join(", ") || job.source || "",
142+
sources: mergedSources,
54143
palavra: job.keyword || job.palavraChave || job.palavra || "",
55-
keywords: [
56-
...new Set(
57-
[job.keyword, job.palavraChave, ...(job.keywords || [])].filter(
58-
Boolean,
59-
),
60-
),
61-
],
144+
keywords: mergedKeywords,
62145
});
63146
}
64147

‎backend/tests/unit/services/pipeline/scrapeAllSources.test.js‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,41 @@ describe("scrapeAllSources", () => {
160160
expect(jobs[0].keywords).toEqual(["React", "Node"]);
161161
});
162162

163+
it("deduplica a mesma vaga entre plataformas pelo título, empresa e local", async () => {
164+
const adapters = [
165+
{
166+
sourceName: "LinkedIn",
167+
search: vi.fn().mockResolvedValue([
168+
{
169+
titulo: "Desenvolvedor(a) Java Jr",
170+
empresa: "Sankhya",
171+
local: "Brasil",
172+
link: "https://www.linkedin.com/jobs/view/desenvolvedor-java-jr-123?refId=abc&trackingId=123",
173+
},
174+
]),
175+
},
176+
{
177+
sourceName: "Greenhouse",
178+
search: vi.fn().mockResolvedValue([
179+
{
180+
titulo: "desenvolvedor a java jr",
181+
empresa: "Sankhya",
182+
local: "Brasil",
183+
link: "https://boards.greenhouse.io/sankhya/jobs/123",
184+
},
185+
]),
186+
},
187+
];
188+
189+
const jobs = await scrapeAllSources(adapters, { keywords: ["Java"] });
190+
191+
expect(jobs).toHaveLength(1);
192+
expect(jobs[0]).toMatchObject({
193+
empresa: "Sankhya",
194+
local: "Brasil",
195+
});
196+
});
197+
163198
it("ignora retorno inválido do adapter e chama logWarn", async () => {
164199
const adapters = [
165200
{

‎frontend/src/hooks/useJobsFiltering.ts‎

Lines changed: 124 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,135 @@
11
import type { Job } from "@/types/jobs";
22
import { useMemo, useState } from "react";
33

4+
function normalizeComparableText(value?: string | null) {
5+
return String(value || "")
6+
.normalize("NFD")
7+
.replace(/\p{Diacritic}/gu, "")
8+
.toLowerCase()
9+
.replace(/[^\p{L}\p{N}]+/gu, " ")
10+
.trim()
11+
.replace(/\s+/g, " ");
12+
}
13+
14+
function normalizeComparableLink(value?: string | null) {
15+
const rawValue = String(value || "").trim();
16+
17+
if (!rawValue) {
18+
return "";
19+
}
20+
21+
try {
22+
const parsedUrl = new URL(rawValue);
23+
parsedUrl.search = "";
24+
parsedUrl.hash = "";
25+
return `${parsedUrl.origin}${parsedUrl.pathname}`.replace(/\/+$/, "");
26+
} catch {
27+
return rawValue.split(/[?#]/)[0].replace(/\/+$/, "");
28+
}
29+
}
30+
31+
function splitJobKeywords(job: Job) {
32+
return [
33+
...new Set(
34+
[
35+
...(Array.isArray(job.keywords) ? job.keywords : []),
36+
...String(job.palavra || "")
37+
.split(/[,;|]+/)
38+
.map((keyword) => keyword.trim()),
39+
].filter(Boolean),
40+
),
41+
];
42+
}
43+
44+
function pickPreferredValue(...values: Array<string | null | undefined>) {
45+
return (
46+
values
47+
.map((value) => String(value || "").trim())
48+
.filter(Boolean)
49+
.sort((left, right) => right.length - left.length)[0] || ""
50+
);
51+
}
52+
53+
function buildDedupKey(job: Job) {
54+
const title = normalizeComparableText(job.titulo);
55+
const company = normalizeComparableText(job.empresa);
56+
const location = normalizeComparableText(job.local);
57+
58+
if (title && company && location) {
59+
return `identity:${title}|${company}|${location}`;
60+
}
61+
62+
if (title && company) {
63+
return `identity:${title}|${company}|${location || "sem-local"}`;
64+
}
65+
66+
const link = normalizeComparableLink(job.link);
67+
68+
if (link) {
69+
return `url:${link}`;
70+
}
71+
72+
return `fallback:${title}|${company}|${location}|${normalizeComparableText(job.source)}`;
73+
}
74+
75+
function dedupeJobs(jobs: Job[]) {
76+
const unique = new Map<string, Job>();
77+
78+
for (const job of jobs) {
79+
const key = buildDedupKey(job);
80+
const existing = unique.get(key);
81+
82+
if (!existing) {
83+
unique.set(key, {
84+
...job,
85+
palavra: splitJobKeywords(job).join(", "),
86+
});
87+
continue;
88+
}
89+
90+
const mergedKeywords = [...new Set([...splitJobKeywords(existing), ...splitJobKeywords(job)])];
91+
const mergedSources = [
92+
...new Set(
93+
[...(existing.sources ?? []), ...(job.sources ?? []), existing.source, job.source]
94+
.map((source) => String(source || "").trim())
95+
.filter(Boolean),
96+
),
97+
];
98+
99+
unique.set(key, {
100+
...existing,
101+
...job,
102+
titulo: pickPreferredValue(existing.titulo, job.titulo),
103+
empresa: pickPreferredValue(existing.empresa, job.empresa),
104+
local: pickPreferredValue(existing.local, job.local),
105+
link: pickPreferredValue(existing.link, job.link),
106+
source: mergedSources.join(", ") || existing.source || job.source || "",
107+
palavra: mergedKeywords.join(", "),
108+
keywords: mergedKeywords,
109+
sources: mergedSources.length > 0 ? mergedSources : undefined,
110+
});
111+
}
112+
113+
return [...unique.values()];
114+
}
115+
4116
export function useJobsFiltering(jobs: Job[]) {
5117
const [search, setSearch] = useState("");
6118
const [keywordFilter, setKeywordFilter] = useState<string[]>([]);
7119

120+
const dedupedJobs = useMemo(() => dedupeJobs(jobs), [jobs]);
121+
8122
const keywords = useMemo(() => {
9-
const values = Array.from(new Set(jobs.map((job) => String(job.palavra || "").trim()).filter(Boolean)));
123+
const values = Array.from(new Set(dedupedJobs.flatMap((job) => splitJobKeywords(job))));
10124
return values.sort((a, b) => a.localeCompare(b));
11-
}, [jobs]);
125+
}, [dedupedJobs]);
12126

13127
const filteredJobs = useMemo(() => {
14-
const term = search.trim().toLowerCase();
128+
const term = normalizeComparableText(search);
15129

16-
return jobs.filter((job) => {
17-
const currentKeyword = String(job.palavra || "").trim();
18-
const byKeyword = keywordFilter.length === 0 || keywordFilter.includes(currentKeyword);
130+
return dedupedJobs.filter((job) => {
131+
const currentKeywords = splitJobKeywords(job);
132+
const byKeyword = keywordFilter.length === 0 || keywordFilter.some((keyword) => currentKeywords.includes(keyword));
19133
if (!byKeyword) {
20134
return false;
21135
}
@@ -24,13 +138,13 @@ export function useJobsFiltering(jobs: Job[]) {
24138
return true;
25139
}
26140

27-
const text = [job.titulo, job.empresa, job.local, job.link, job.palavra]
28-
.map((value) => String(value || "").toLowerCase())
29-
.join(" ");
141+
const text = normalizeComparableText(
142+
[job.titulo, job.empresa, job.local, job.link, job.palavra, ...(job.keywords || [])].join(" "),
143+
);
30144

31145
return text.includes(term);
32146
});
33-
}, [jobs, search, keywordFilter]);
147+
}, [dedupedJobs, search, keywordFilter]);
34148

35149
return {
36150
search,

‎frontend/src/types/jobs.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
export interface Job {
22
palavra?: string | null;
3+
keywords?: string[] | null;
34
titulo?: string | null;
45
empresa?: string | null;
56
source?: string | null;
7+
sources?: string[] | null;
68
local?: string | null;
79
link?: string | null;
810
}

0 commit comments

Comments
 (0)