feat: secure redis env config for VPS (#75)

Author: Benevanio

README.md

@@ -134,6 +134,29 @@ npm run dev:frontend
 npm run dev:backend
 ```
 
+## Configuracao segura na VPS (Redis / Postgres)
+
+Para ambiente de VPS e projeto open source, **nao coloque `DATABASE_URL` nem `REDIS_URL` em `environment.json`**.
+Use somente `backend/.env` no servidor (arquivo ignorado pelo Git) e mantenha `KEYWORDS_STORAGE_MODE=env`.
+
+Exemplo seguro:
+
+```env
+KEYWORDS_STORAGE_MODE=env
+SEARCH_KEYWORDS=Java,Spring,RabbitMQ,Docker
+DATABASE_URL=postgresql://app_user:senha_forte@127.0.0.1:5432/jobsglobalscraper
+REDIS_URL=redis://:senha_forte@127.0.0.1:6379/0
+REDIS_KEY_PREFIX=vagas-full
+CACHE_TTL_MS=600000
+```
+
+Recomendacoes:
+
+- use `127.0.0.1` ou a rede interna do Docker na VPS, em vez de expor Redis publicamente
+- prefira `rediss://` se o Redis estiver fora da maquina local/VPS
+- mantenha a porta `6379` fechada para acesso externo no firewall
+- com `KEYWORDS_STORAGE_MODE=env`, o backend deixa de depender de `backend/src/db/environment.json` em producao
+
 ## Scripts principais
 
 ### Raiz

backend/.env.example

@@ -3,7 +3,7 @@ NODE_ENV=production
 PORT=3001
 
 # CORS / frontend access
-CORS_ALLOWED_ORIGINS=https://painel-vagas-lake.vercel.app,http://localhost:5173,http://localhost:5174
+CORS_ALLOWED_ORIGINS=https://painel-vagas-lake.vercel.app,https://painel-vagas-m6hbzlqeh-bene-teslas-projects.vercel.app,http://localhost:5173,http://localhost:5174,http://jobsglobalscraper.ddns.net,https://jobsglobalscraper.ddns.net
 
 # LinkedIn search filters
 SEARCH_LOCATION=Brasil
@@ -12,8 +12,20 @@ SEARCH_LANGUAGE=pt
 REMOTE_ONLY=true
 JOB_TYPES=C,F
 TIME_FILTER=r604800
+
+# Keywords storage:
+# - redis => fonte de verdade para salvar e ler keywords
+# SEARCH_KEYWORDS fica apenas como fallback inicial se a chave ainda estiver vazia no Redis
+KEYWORDS_STORAGE_MODE=redis
 SEARCH_KEYWORDS=UX Designer,UI Designer,Product Manager,Product Owner
 
+# External services (nunca commitar valores reais)
+DATABASE_URL=postgresql://app_user:change_me@127.0.0.1:5432/jobsglobalscraper
+REDIS_URL=redis://:change_me@127.0.0.1:6379/0
+REDIS_KEY_PREFIX=vagas-full
+KEYWORDS_REDIS_KEY=vagas-full:keywords
+CACHE_TTL_MS=600000
+
 # Scraping behavior
 WAIT_BETWEEN_SEARCHES_MS=5000
 PAGE_TIMEOUT_MS=10000

backend/package.json

@@ -37,6 +37,7 @@
     "dotenv": "^17.3.1",
     "express": "^5.2.1",
     "pdfkit": "^0.18.0",
+    "redis": "^4.7.1",
     "swagger-jsdoc": "^6.2.8",
     "swagger-ui-express": "^5.0.1",
     "xlsx": "^0.18.5"

backend/src/app.js

@@ -1,11 +1,16 @@
 import { getConfig } from "./config.js";
+import { loadKeywords } from "./db/keywordsStore.js";
 import { exportToExcel, exportToPDF } from "./exporter.js";
 import { logInfo } from "./logger.js";
 import { searchJobsWithCache } from "./pipeline/searchJobsWithCache.js";
 import { sources } from "./sources/index.js";
 
 export async function run() {
-  const config = getConfig();
+  const baseConfig = getConfig();
+  const config = {
+    ...baseConfig,
+    keywords: await loadKeywords(baseConfig.keywords),
+  };
 
   logInfo(`Localização da busca: ${config.searchLocation}`);
   logInfo(`Total de palavras-chave: ${config.keywords.length}`);

backend/src/cache/cache.js

@@ -1,3 +1,131 @@
+let redisClientPromise = null;
+let redisClientUrl = "";
+let redisWarningShown = false;
+
+const cacheStatus = {
+  provider: "memory",
+  configured: false,
+  connected: false,
+  lastError: null,
+};
+
+function readRedisUrl() {
+  return process.env.REDIS_URL?.trim() || "";
+}
+
+function syncCacheStatus() {
+  const redisUrl = readRedisUrl();
+  cacheStatus.configured = Boolean(redisUrl);
+  cacheStatus.provider = redisUrl ? "redis" : "memory";
+
+  if (!redisUrl) {
+    cacheStatus.connected = false;
+    cacheStatus.lastError = null;
+  }
+}
+
+function warnRedisFallback(message, error) {
+  cacheStatus.connected = false;
+  cacheStatus.lastError = error instanceof Error ? error.message : null;
+
+  if (redisWarningShown) {
+    return;
+  }
+
+  redisWarningShown = true;
+
+  const errorMessage = error instanceof Error ? error.message : "";
+  // eslint-disable-next-line no-console
+  console.warn(`${message}${errorMessage ? ` (${errorMessage})` : ""}`);
+}
+
+export async function getRedisClient() {
+  const redisUrl = readRedisUrl();
+  syncCacheStatus();
+
+  if (!redisUrl) {
+    return null;
+  }
+
+  if (redisClientUrl !== redisUrl) {
+    redisClientPromise = null;
+    redisClientUrl = redisUrl;
+  }
+
+  if (!redisClientPromise) {
+    redisClientPromise = import("redis")
+      .then(async ({ createClient }) => {
+        const client = createClient({
+          url: redisUrl,
+          socket: {
+            connectTimeout: 3_000,
+            reconnectStrategy(retries) {
+              if (retries >= 2) {
+                return false;
+              }
+
+              return Math.min((retries + 1) * 100, 500);
+            },
+          },
+        });
+
+        client.on("ready", () => {
+          cacheStatus.connected = true;
+          cacheStatus.lastError = null;
+        });
+
+        client.on("end", () => {
+          cacheStatus.connected = false;
+        });
+
+        client.on("error", (error) => {
+          warnRedisFallback("Redis indisponivel, usando cache em memoria.", error);
+        });
+
+        await client.connect();
+        return client;
+      })
+      .catch((error) => {
+        warnRedisFallback("Falha ao conectar no Redis, usando cache em memoria.", error);
+        redisClientPromise = null;
+        return null;
+      });
+  }
+
+  return redisClientPromise;
+}
+
+export function getCacheStatus() {
+  syncCacheStatus();
+  return {
+    ...cacheStatus,
+    type: cache.constructor.name,
+  };
+}
+
+export async function warmupCache() {
+  syncCacheStatus();
+
+  if (!cacheStatus.configured) {
+    return getCacheStatus();
+  }
+
+  const client = await getRedisClient();
+  if (!client) {
+    return getCacheStatus();
+  }
+
+  try {
+    const pong = await client.ping();
+    cacheStatus.connected = pong === "PONG";
+    cacheStatus.lastError = cacheStatus.connected ? null : "PING sem resposta esperada";
+  } catch (error) {
+    warnRedisFallback("Falha ao validar conexao com Redis.", error);
+  }
+
+  return getCacheStatus();
+}
+
 export class MemoryCache {
   constructor() {
     this.store = new Map();
@@ -38,4 +166,89 @@ export class MemoryCache {
   }
 }
 
-export const cache = new MemoryCache();
+export class RedisCache {
+  constructor(options = {}) {
+    this.prefix = options.prefix || process.env.REDIS_KEY_PREFIX?.trim() || "vagas-full";
+    this.memoryFallback = new MemoryCache();
+  }
+
+  buildKey(key) {
+    return `${this.prefix}:${key}`;
+  }
+
+  async get(key) {
+    const client = await getRedisClient();
+
+    if (!client) {
+      return this.memoryFallback.get(key);
+    }
+
+    try {
+      const raw = await client.get(this.buildKey(key));
+      return raw ? JSON.parse(raw) : null;
+    } catch (error) {
+      warnRedisFallback("Erro ao ler do Redis, usando cache em memoria.", error);
+      return this.memoryFallback.get(key);
+    }
+  }
+
+  async set(key, value, ttlMs) {
+    this.memoryFallback.set(key, value, ttlMs);
+
+    const client = await getRedisClient();
+    if (!client) {
+      return;
+    }
+
+    try {
+      await client.set(this.buildKey(key), JSON.stringify(value), {
+        PX: Math.max(1, Number(ttlMs) || 1),
+      });
+    } catch (error) {
+      warnRedisFallback("Erro ao salvar no Redis, usando cache em memoria.", error);
+    }
+  }
+
+  async delete(key) {
+    this.memoryFallback.delete(key);
+
+    const client = await getRedisClient();
+    if (!client) {
+      return;
+    }
+
+    try {
+      await client.del(this.buildKey(key));
+    } catch (error) {
+      warnRedisFallback("Erro ao remover chave do Redis.", error);
+    }
+  }
+
+  async clear() {
+    this.memoryFallback.clear();
+
+    const client = await getRedisClient();
+    if (!client) {
+      return;
+    }
+
+    try {
+      const keys = [];
+      for await (const key of client.scanIterator({ MATCH: `${this.prefix}:*` })) {
+        keys.push(key);
+      }
+
+      if (keys.length > 0) {
+        await client.del(keys);
+      }
+    } catch (error) {
+      warnRedisFallback("Erro ao limpar o Redis.", error);
+    }
+  }
+
+  async has(key) {
+    return (await this.get(key)) !== null;
+  }
+}
+
+export const cache = process.env.REDIS_URL?.trim() ? new RedisCache() : new MemoryCache();