Develop

backend/src/db/environment.json

@@ -1,8 +1,5 @@
 {
   "KEYWORDS": [
-    "Java",
-    "Spring",
-    "RabbitMQ",
-    "Docker"
+    "Java"
   ]
 }

backend/src/pipeline/scrapeAllSources.js

@@ -10,55 +10,138 @@ function normalizeJob(job, keyword, adapter) {
   };
 }
 
-function mergeKeywords(existing, incoming) {
-  const keywords = new Set([
-    ...(existing.keywords || []),
-    ...(existing.keyword ? [existing.keyword] : []),
-    ...(existing.palavraChave ? [existing.palavraChave] : []),
-    ...(incoming.keywords || []),
-    ...(incoming.keyword ? [incoming.keyword] : []),
-    ...(incoming.palavraChave ? [incoming.palavraChave] : []),
-  ]);
+function normalizeComparableText(value) {
+  return String(value || "")
+    .normalize("NFD")
+    .replace(/\p{Diacritic}/gu, "")
+    .toLowerCase()
+    .replace(/[^\p{L}\p{N}]+/gu, " ")
+    .trim()
+    .replace(/\s+/g, " ");
+}
+
+function normalizeComparableUrl(value) {
+  const rawValue = String(value || "").trim();
+
+  if (!rawValue) {
+    return "";
+  }
+
+  try {
+    const parsedUrl = new URL(rawValue);
+    parsedUrl.search = "";
+    parsedUrl.hash = "";
+    return `${parsedUrl.origin}${parsedUrl.pathname}`.replace(/\/+$/, "");
+  } catch {
+    return rawValue.split(/[?#]/)[0].replace(/\/+$/, "");
+  }
+}
+
+function getMergedKeywords(...jobs) {
+  return [
+    ...new Set(
+      jobs.flatMap((job) =>
+        [
+          ...(Array.isArray(job.keywords) ? job.keywords : []),
+          job.keyword,
+          job.palavraChave,
+          job.palavra,
+        ]
+          .filter(Boolean)
+          .map((keyword) => String(keyword).trim()),
+      ),
+    ),
+  ];
+}
 
-  const mergedKeywords = [...keywords].filter(Boolean);
+function getMergedSources(...jobs) {
+  return [
+    ...new Set(
+      jobs.flatMap((job) =>
+        [
+          ...(Array.isArray(job.sources) ? job.sources : []),
+          job.source,
+        ]
+          .filter(Boolean)
+          .map((source) => String(source).trim()),
+      ),
+    ),
+  ];
+}
+
+function pickPreferredValue(...values) {
+  return (
+    values
+      .map((value) => String(value || "").trim())
+      .filter(Boolean)
+      .sort((left, right) => right.length - left.length)[0] || ""
+  );
+}
+
+function mergeKeywords(existing, incoming) {
+  const mergedKeywords = getMergedKeywords(existing, incoming);
+  const mergedSources = getMergedSources(existing, incoming);
 
   return {
     ...existing,
     ...incoming,
+    titulo: pickPreferredValue(existing.titulo, incoming.titulo, existing.title, incoming.title),
+    empresa: pickPreferredValue(existing.empresa, incoming.empresa, existing.company, incoming.company),
+    local: pickPreferredValue(existing.local, incoming.local, existing.location, incoming.location),
+    link: pickPreferredValue(existing.link, incoming.link, existing.jobUrl, incoming.jobUrl),
+    jobUrl: pickPreferredValue(existing.jobUrl, incoming.jobUrl, existing.link, incoming.link),
+    source: mergedSources.join(", ") || existing.source || incoming.source || "",
+    sources: mergedSources,
     keyword: mergedKeywords[0] || "",
     palavraChave: mergedKeywords[0] || "",
     keywords: mergedKeywords,
     palavra: mergedKeywords[0] || "",
   };
 }
 
+function buildDedupKey(job) {
+  const title = normalizeComparableText(job.titulo || job.title);
+  const company = normalizeComparableText(job.empresa || job.company);
+  const location = normalizeComparableText(job.local || job.location);
+
+  if (title && company && location) {
+    return `identity:${title}|${company}|${location}`;
+  }
+
+  if (title && company) {
+    return `identity:${title}|${company}|${location || "sem-local"}`;
+  }
+
+  const normalizedLink = normalizeComparableUrl(job.link || job.jobUrl);
+
+  if (normalizedLink) {
+    return `url:${normalizedLink}`;
+  }
+
+  return `fallback:${title}|${company}|${location}|${normalizeComparableText(job.source)}`;
+}
+
 function dedupeJobs(jobs) {
   const unique = new Map();
 
   for (const job of jobs) {
-    const key =
-      job.link ||
-      job.jobUrl ||
-      `${job.source}-${job.titulo || job.title}-${job.empresa || job.company}-${job.local || job.location}`;
-
-    if (!key) continue;
+    const key = buildDedupKey(job);
 
     if (unique.has(key)) {
       const existing = unique.get(key);
       unique.set(key, mergeKeywords(existing, job));
       continue;
     }
 
+    const mergedKeywords = getMergedKeywords(job);
+    const mergedSources = getMergedSources(job);
+
     unique.set(key, {
       ...job,
+      source: mergedSources.join(", ") || job.source || "",
+      sources: mergedSources,
       palavra: job.keyword || job.palavraChave || job.palavra || "",
-      keywords: [
-        ...new Set(
-          [job.keyword, job.palavraChave, ...(job.keywords || [])].filter(
-            Boolean,
-          ),
-        ),
-      ],
+      keywords: mergedKeywords,
     });
   }
 

backend/tests/unit/services/pipeline/scrapeAllSources.test.js

@@ -160,6 +160,41 @@ describe("scrapeAllSources", () => {
     expect(jobs[0].keywords).toEqual(["React", "Node"]);
   });
 
+  it("deduplica a mesma vaga entre plataformas pelo título, empresa e local", async () => {
+    const adapters = [
+      {
+        sourceName: "LinkedIn",
+        search: vi.fn().mockResolvedValue([
+          {
+            titulo: "Desenvolvedor(a) Java Jr",
+            empresa: "Sankhya",
+            local: "Brasil",
+            link: "https://www.linkedin.com/jobs/view/desenvolvedor-java-jr-123?refId=abc&trackingId=123",
+          },
+        ]),
+      },
+      {
+        sourceName: "Greenhouse",
+        search: vi.fn().mockResolvedValue([
+          {
+            titulo: "desenvolvedor a java jr",
+            empresa: "Sankhya",
+            local: "Brasil",
+            link: "https://boards.greenhouse.io/sankhya/jobs/123",
+          },
+        ]),
+      },
+    ];
+
+    const jobs = await scrapeAllSources(adapters, { keywords: ["Java"] });
+
+    expect(jobs).toHaveLength(1);
+    expect(jobs[0]).toMatchObject({
+      empresa: "Sankhya",
+      local: "Brasil",
+    });
+  });
+
   it("ignora retorno inválido do adapter e chama logWarn", async () => {
     const adapters = [
       {

frontend/src/hooks/useJobsFiltering.ts

@@ -1,21 +1,135 @@
 import type { Job } from "@/types/jobs";
 import { useMemo, useState } from "react";
 
+function normalizeComparableText(value?: string | null) {
+  return String(value || "")
+    .normalize("NFD")
+    .replace(/\p{Diacritic}/gu, "")
+    .toLowerCase()
+    .replace(/[^\p{L}\p{N}]+/gu, " ")
+    .trim()
+    .replace(/\s+/g, " ");
+}
+
+function normalizeComparableLink(value?: string | null) {
+  const rawValue = String(value || "").trim();
+
+  if (!rawValue) {
+    return "";
+  }
+
+  try {
+    const parsedUrl = new URL(rawValue);
+    parsedUrl.search = "";
+    parsedUrl.hash = "";
+    return `${parsedUrl.origin}${parsedUrl.pathname}`.replace(/\/+$/, "");
+  } catch {
+    return rawValue.split(/[?#]/)[0].replace(/\/+$/, "");
+  }
+}
+
+function splitJobKeywords(job: Job) {
+  return [
+    ...new Set(
+      [
+        ...(Array.isArray(job.keywords) ? job.keywords : []),
+        ...String(job.palavra || "")
+          .split(/[,;|]+/)
+          .map((keyword) => keyword.trim()),
+      ].filter(Boolean),
+    ),
+  ];
+}
+
+function pickPreferredValue(...values: Array<string | null | undefined>) {
+  return (
+    values
+      .map((value) => String(value || "").trim())
+      .filter(Boolean)
+      .sort((left, right) => right.length - left.length)[0] || ""
+  );
+}
+
+function buildDedupKey(job: Job) {
+  const title = normalizeComparableText(job.titulo);
+  const company = normalizeComparableText(job.empresa);
+  const location = normalizeComparableText(job.local);
+
+  if (title && company && location) {
+    return `identity:${title}|${company}|${location}`;
+  }
+
+  if (title && company) {
+    return `identity:${title}|${company}|${location || "sem-local"}`;
+  }
+
+  const link = normalizeComparableLink(job.link);
+
+  if (link) {
+    return `url:${link}`;
+  }
+
+  return `fallback:${title}|${company}|${location}|${normalizeComparableText(job.source)}`;
+}
+
+function dedupeJobs(jobs: Job[]) {
+  const unique = new Map<string, Job>();
+
+  for (const job of jobs) {
+    const key = buildDedupKey(job);
+    const existing = unique.get(key);
+
+    if (!existing) {
+      unique.set(key, {
+        ...job,
+        palavra: splitJobKeywords(job).join(", "),
+      });
+      continue;
+    }
+
+    const mergedKeywords = [...new Set([...splitJobKeywords(existing), ...splitJobKeywords(job)])];
+    const mergedSources = [
+      ...new Set(
+        [...(existing.sources ?? []), ...(job.sources ?? []), existing.source, job.source]
+          .map((source) => String(source || "").trim())
+          .filter(Boolean),
+      ),
+    ];
+
+    unique.set(key, {
+      ...existing,
+      ...job,
+      titulo: pickPreferredValue(existing.titulo, job.titulo),
+      empresa: pickPreferredValue(existing.empresa, job.empresa),
+      local: pickPreferredValue(existing.local, job.local),
+      link: pickPreferredValue(existing.link, job.link),
+      source: mergedSources.join(", ") || existing.source || job.source || "",
+      palavra: mergedKeywords.join(", "),
+      keywords: mergedKeywords,
+      sources: mergedSources.length > 0 ? mergedSources : undefined,
+    });
+  }
+
+  return [...unique.values()];
+}
+
 export function useJobsFiltering(jobs: Job[]) {
   const [search, setSearch] = useState("");
   const [keywordFilter, setKeywordFilter] = useState<string[]>([]);
 
+  const dedupedJobs = useMemo(() => dedupeJobs(jobs), [jobs]);
+
   const keywords = useMemo(() => {
-    const values = Array.from(new Set(jobs.map((job) => String(job.palavra || "").trim()).filter(Boolean)));
+    const values = Array.from(new Set(dedupedJobs.flatMap((job) => splitJobKeywords(job))));
     return values.sort((a, b) => a.localeCompare(b));
-  }, [jobs]);
+  }, [dedupedJobs]);
 
   const filteredJobs = useMemo(() => {
-    const term = search.trim().toLowerCase();
+    const term = normalizeComparableText(search);
 
-    return jobs.filter((job) => {
-      const currentKeyword = String(job.palavra || "").trim();
-      const byKeyword = keywordFilter.length === 0 || keywordFilter.includes(currentKeyword);
+    return dedupedJobs.filter((job) => {
+      const currentKeywords = splitJobKeywords(job);
+      const byKeyword = keywordFilter.length === 0 || keywordFilter.some((keyword) => currentKeywords.includes(keyword));
       if (!byKeyword) {
         return false;
       }
@@ -24,13 +138,13 @@ export function useJobsFiltering(jobs: Job[]) {
         return true;
       }
 
-      const text = [job.titulo, job.empresa, job.local, job.link, job.palavra]
-        .map((value) => String(value || "").toLowerCase())
-        .join(" ");
+      const text = normalizeComparableText(
+        [job.titulo, job.empresa, job.local, job.link, job.palavra, ...(job.keywords || [])].join(" "),
+      );
 
       return text.includes(term);
     });
-  }, [jobs, search, keywordFilter]);
+  }, [dedupedJobs, search, keywordFilter]);
 
   return {
     search,

frontend/src/types/jobs.ts

@@ -1,8 +1,10 @@
 export interface Job {
   palavra?: string | null;
+  keywords?: string[] | null;
   titulo?: string | null;
   empresa?: string | null;
   source?: string | null;
+  sources?: string[] | null;
   local?: string | null;
   link?: string | null;
 }