Skip to content

Commit 41cdc80

Browse files
BenevanioBenevanio
committed
feat: update keywords storage to use Redis as primary source and adjust related configurations
1 parent 80838b5 commit 41cdc80

5 files changed

Lines changed: 62 additions & 199 deletions

File tree

‎backend/.env.example‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,9 @@ JOB_TYPES=C,F
1414
TIME_FILTER=r604800
1515

1616
# Keywords storage:
17-
# - env => recomendado para VPS / Docker / projeto open source
18-
# - file => mantém compatibilidade com backend/src/db/environment.json
19-
KEYWORDS_STORAGE_MODE=env
17+
# - redis => fonte de verdade para salvar e ler keywords
18+
# SEARCH_KEYWORDS fica apenas como fallback inicial se a chave ainda estiver vazia no Redis
19+
KEYWORDS_STORAGE_MODE=redis
2020
SEARCH_KEYWORDS=UX Designer,UI Designer,Product Manager,Product Owner
2121

2222
# External services (nunca commitar valores reais)

‎backend/src/config.js‎

Lines changed: 14 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,4 @@
1-
import { existsSync, readFileSync } from "fs";
2-
import path from "path";
3-
import { fileURLToPath } from "url";
4-
5-
const MODULE_DIR = path.dirname(fileURLToPath(import.meta.url));
6-
7-
const DEFAULT_KEYWORDS = [
1+
const DEFAULT_KEYWORDS = [
82
"Java",
93
"JavaScript",
104
"React",
@@ -39,18 +33,11 @@ function parseNumber(value, fallback) {
3933
}
4034

4135
function getKeywordsStorageMode() {
42-
const configuredMode = String(process.env.KEYWORDS_STORAGE_MODE ?? "file")
36+
const configuredMode = String(process.env.KEYWORDS_STORAGE_MODE ?? "redis")
4337
.trim()
4438
.toLowerCase();
4539

46-
return configuredMode === "env" ? "env" : "file";
47-
}
48-
49-
function getKeywordsFilePath() {
50-
const configuredPath = process.env.KEYWORDS_FILE_PATH?.trim();
51-
return configuredPath
52-
? path.resolve(configuredPath)
53-
: path.resolve(MODULE_DIR, "db", "environment.json");
40+
return configuredMode === "env" ? "env" : "redis";
5441
}
5542

5643
function normalizeKeywords(keywords) {
@@ -62,46 +49,18 @@ function normalizeKeywords(keywords) {
6249
}
6350

6451
function parseKeywordsFromEnv(value) {
65-
if (!value) {
66-
return null;
67-
}
68-
69-
const keywords = String(value)
70-
.split(",")
71-
.map((item) => item.trim())
72-
.filter(Boolean);
73-
74-
return keywords.length > 0 ? keywords : null;
52+
const keywords = normalizeKeywords(
53+
String(value ?? "")
54+
.split(",")
55+
.map((item) => item.trim())
56+
.filter(Boolean),
57+
);
58+
59+
return keywords?.length ? keywords : null;
7560
}
7661

7762
function parseKeywords(value) {
78-
const keywordsStorageMode = getKeywordsStorageMode();
79-
80-
if (keywordsStorageMode === "env") {
81-
const keywordsFromEnv = parseKeywordsFromEnv(value);
82-
if (keywordsFromEnv) {
83-
return keywordsFromEnv;
84-
}
85-
}
86-
87-
try {
88-
const envPath = getKeywordsFilePath();
89-
if (existsSync(envPath)) {
90-
const data = JSON.parse(readFileSync(envPath, "utf-8"));
91-
if (Array.isArray(data.KEYWORDS)) {
92-
return normalizeKeywords(data.KEYWORDS) ?? [];
93-
}
94-
}
95-
} catch {
96-
// Se falhar, fallback
97-
}
98-
99-
const keywordsFromEnv = parseKeywordsFromEnv(value);
100-
if (keywordsFromEnv) {
101-
return keywordsFromEnv;
102-
}
103-
104-
return DEFAULT_KEYWORDS;
63+
return parseKeywordsFromEnv(value) ?? DEFAULT_KEYWORDS;
10564
}
10665

10766
function parseTimeFilter(value, fallback) {
@@ -121,7 +80,7 @@ export function getConfig() {
12180
maxPagesPerKeyword: parseNumber(process.env.MAX_PAGES_PER_KEYWORD, 5),
12281
viewport: {
12382
width: parseNumber(process.env.VIEWPORT_WIDTH, 1280),
124-
height: parseNumber(process.env.VIEWPORT_HEIGHT, 800)
83+
height: parseNumber(process.env.VIEWPORT_HEIGHT, 800),
12584
},
12685
outputFile: process.env.OUTPUT_FILE || "output/vagas_remoto.xlsx",
12786
pdfFile: process.env.PDF_FILE || "output/vagas_remoto.pdf",
@@ -137,6 +96,6 @@ export function getConfig() {
13796
cacheTtlMs: parseNumber(process.env.CACHE_TTL_MS, 10 * 60 * 1000),
13897
databaseUrl: process.env.DATABASE_URL?.trim() || "",
13998
redisUrl: process.env.REDIS_URL?.trim() || "",
140-
redisKeyPrefix: process.env.REDIS_KEY_PREFIX?.trim() || "vagas-full"
99+
redisKeyPrefix: process.env.REDIS_KEY_PREFIX?.trim() || "vagas-full",
141100
};
142101
}

‎backend/src/db/keywordsStore.js‎

Lines changed: 28 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,5 @@
1-
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
2-
import path from "path";
3-
import { fileURLToPath } from "url";
41
import { getRedisClient } from "../cache/cache.js";
52

6-
const MODULE_DIR = path.dirname(fileURLToPath(import.meta.url));
7-
8-
function getKeywordsStorageMode() {
9-
const configuredMode = String(process.env.KEYWORDS_STORAGE_MODE ?? "file")
10-
.trim()
11-
.toLowerCase();
12-
13-
return configuredMode === "env" ? "env" : "file";
14-
}
15-
16-
function getKeywordsFilePath() {
17-
const configuredPath = process.env.KEYWORDS_FILE_PATH?.trim();
18-
return configuredPath
19-
? path.resolve(configuredPath)
20-
: path.resolve(MODULE_DIR, "environment.json");
21-
}
22-
233
function getKeywordsRedisKey() {
244
const configuredKey = process.env.KEYWORDS_REDIS_KEY?.trim();
255
if (configuredKey) {
@@ -47,60 +27,43 @@ function parseKeywordsFromEnv(value) {
4727
) ?? [];
4828
}
4929

50-
function readKeywordsFromFile() {
51-
const envPath = getKeywordsFilePath();
52-
53-
if (!existsSync(envPath)) {
54-
return null;
55-
}
56-
57-
try {
58-
const data = JSON.parse(readFileSync(envPath, "utf-8"));
59-
return Array.isArray(data?.KEYWORDS) ? normalizeKeywords(data.KEYWORDS) ?? [] : [];
60-
} catch {
61-
return [];
30+
function getFallbackKeywords(fallback = []) {
31+
const envKeywords = parseKeywordsFromEnv(process.env.SEARCH_KEYWORDS);
32+
if (envKeywords.length > 0) {
33+
return envKeywords;
6234
}
63-
}
6435

65-
function writeKeywordsToFile(keywords) {
66-
const envPath = getKeywordsFilePath();
67-
mkdirSync(path.dirname(envPath), { recursive: true });
68-
writeFileSync(envPath, JSON.stringify({ KEYWORDS: keywords }, null, 2), "utf-8");
36+
return normalizeKeywords(fallback) ?? [];
6937
}
7038

7139
export async function loadKeywords(fallback = []) {
40+
const fallbackKeywords = getFallbackKeywords(fallback);
7241
const client = await getRedisClient();
7342

74-
if (client) {
75-
try {
76-
const raw = await client.get(getKeywordsRedisKey());
77-
if (raw) {
78-
const parsed = JSON.parse(raw);
79-
80-
if (Array.isArray(parsed)) {
81-
return normalizeKeywords(parsed) ?? [];
82-
}
83-
84-
if (Array.isArray(parsed?.KEYWORDS)) {
85-
return normalizeKeywords(parsed.KEYWORDS) ?? [];
86-
}
87-
}
88-
} catch {
89-
// fallback below
90-
}
43+
if (!client) {
44+
return fallbackKeywords;
9145
}
9246

93-
const envKeywords = parseKeywordsFromEnv(process.env.SEARCH_KEYWORDS);
94-
if (envKeywords.length > 0) {
95-
return envKeywords;
96-
}
47+
try {
48+
const raw = await client.get(getKeywordsRedisKey());
49+
if (!raw) {
50+
return fallbackKeywords;
51+
}
9752

98-
const fileKeywords = readKeywordsFromFile();
99-
if (fileKeywords !== null) {
100-
return fileKeywords;
53+
const parsed = JSON.parse(raw);
54+
55+
if (Array.isArray(parsed)) {
56+
return normalizeKeywords(parsed) ?? [];
57+
}
58+
59+
if (Array.isArray(parsed?.KEYWORDS)) {
60+
return normalizeKeywords(parsed.KEYWORDS) ?? [];
61+
}
62+
} catch {
63+
// fallback below
10164
}
10265

103-
return normalizeKeywords(fallback) ?? [];
66+
return fallbackKeywords;
10467
}
10568

10669
export async function saveKeywords(keywords) {
@@ -111,16 +74,10 @@ export async function saveKeywords(keywords) {
11174
}
11275

11376
const client = await getRedisClient();
114-
if (client) {
115-
await client.set(getKeywordsRedisKey(), JSON.stringify(normalizedKeywords));
116-
return normalizedKeywords;
117-
}
118-
119-
if (getKeywordsStorageMode() === "env") {
120-
process.env.SEARCH_KEYWORDS = normalizedKeywords.join(",");
121-
return normalizedKeywords;
77+
if (!client) {
78+
throw new Error("Redis indisponivel para salvar keywords.");
12279
}
12380

124-
writeKeywordsToFile(normalizedKeywords);
81+
await client.set(getKeywordsRedisKey(), JSON.stringify(normalizedKeywords));
12582
return normalizedKeywords;
12683
}

‎backend/tests/unit/services/db/keywordsStore.test.js‎

Lines changed: 9 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,9 @@
11
import { beforeEach, describe, expect, it, vi } from "vitest";
22

33
const mocks = vi.hoisted(() => ({
4-
existsSync: vi.fn(),
5-
mkdirSync: vi.fn(),
6-
readFileSync: vi.fn(),
7-
writeFileSync: vi.fn(),
84
getRedisClient: vi.fn(),
95
}));
106

11-
vi.mock("fs", () => ({
12-
existsSync: mocks.existsSync,
13-
mkdirSync: mocks.mkdirSync,
14-
readFileSync: mocks.readFileSync,
15-
writeFileSync: mocks.writeFileSync,
16-
}));
17-
187
vi.mock("../../../../src/cache/cache.js", () => ({
198
getRedisClient: mocks.getRedisClient,
209
}));
@@ -33,7 +22,6 @@ describe("keywordsStore", () => {
3322
delete process.env.REDIS_KEY_PREFIX;
3423
delete process.env.KEYWORDS_FILE_PATH;
3524
mocks.getRedisClient.mockResolvedValue(null);
36-
mocks.existsSync.mockReturnValue(false);
3725
});
3826

3927
it("normaliza keywords removendo vazios e duplicadas", async () => {
@@ -80,27 +68,18 @@ describe("keywordsStore", () => {
8068
await expect(loadKeywords(["fallback"])) .resolves.toEqual(["Suporte", "Node"]);
8169
});
8270

83-
it("lê do arquivo quando env está vazio e retorna [] para JSON inválido", async () => {
84-
process.env.KEYWORDS_FILE_PATH = "./tmp/keywords.json";
85-
mocks.existsSync.mockReturnValue(true);
86-
mocks.readFileSync.mockReturnValueOnce(JSON.stringify({ KEYWORDS: ["QA", "QA", "Node"] }));
87-
88-
const { loadKeywords } = await importKeywordsStore();
89-
90-
await expect(loadKeywords(["fallback"])) .resolves.toEqual(["QA", "Node"]);
91-
expect(mocks.readFileSync).toHaveBeenCalledWith(expect.stringContaining("keywords.json"), "utf-8");
92-
93-
mocks.readFileSync.mockReturnValueOnce("{invalid-json");
94-
await expect(loadKeywords(["fallback"])) .resolves.toEqual([]);
95-
});
96-
97-
it("usa o fallback recebido quando não encontra Redis, env nem arquivo", async () => {
71+
it("usa o fallback recebido quando Redis está indisponível ou retorna JSON inválido", async () => {
9872
const { loadKeywords } = await importKeywordsStore();
9973

10074
await expect(loadKeywords([" Produto ", "Produto", "Dados "])) .resolves.toEqual([
10175
"Produto",
10276
"Dados",
10377
]);
78+
79+
const client = { get: vi.fn().mockResolvedValue("{invalid-json") };
80+
mocks.getRedisClient.mockResolvedValue(client);
81+
82+
await expect(loadKeywords(["QA", "Dados"])) .resolves.toEqual(["QA", "Dados"]);
10483
});
10584

10685
it("salva no Redis usando a chave configurada", async () => {
@@ -114,28 +93,12 @@ describe("keywordsStore", () => {
11493
expect(client.set).toHaveBeenCalledWith("vagas-full:keywords", JSON.stringify(["Node", "React"]));
11594
});
11695

117-
it("salva em SEARCH_KEYWORDS quando está em modo env e não há Redis", async () => {
118-
process.env.KEYWORDS_STORAGE_MODE = "env";
119-
96+
it("retorna null para payload inválido e falha quando Redis está indisponível", async () => {
12097
const { saveKeywords } = await importKeywordsStore();
12198

122-
await expect(saveKeywords(["Backend", "Java", "Backend"])) .resolves.toEqual(["Backend", "Java"]);
123-
expect(process.env.SEARCH_KEYWORDS).toBe("Backend,Java");
12499
await expect(saveKeywords("invalido")).resolves.toBeNull();
125-
});
126-
127-
it("salva em arquivo quando está em modo file e não há Redis", async () => {
128-
process.env.KEYWORDS_STORAGE_MODE = "file";
129-
process.env.KEYWORDS_FILE_PATH = "./tmp/file-keywords.json";
130-
131-
const { saveKeywords } = await importKeywordsStore();
132-
133-
await expect(saveKeywords(["SRE", "Platform"])) .resolves.toEqual(["SRE", "Platform"]);
134-
expect(mocks.mkdirSync).toHaveBeenCalledWith(expect.stringContaining("tmp"), { recursive: true });
135-
expect(mocks.writeFileSync).toHaveBeenCalledWith(
136-
expect.stringContaining("file-keywords.json"),
137-
JSON.stringify({ KEYWORDS: ["SRE", "Platform"] }, null, 2),
138-
"utf-8",
100+
await expect(saveKeywords(["Backend", "Java"])) .rejects.toThrow(
101+
"Redis indisponivel para salvar keywords.",
139102
);
140103
});
141104
});

0 commit comments

Comments
 (0)