Feat/observability stack (#151)

## O que mudou

Este PR contém duas mudanças relacionadas:

### 1. Stack de observabilidade (Prometheus, Grafana, Loki)
- Instrumenta backend (Express) e scraper-go com métricas Prometheus
customizadas
- Adiciona exporters para Postgres, Redis/Valkey, host e containers
- Substitui ELK por Loki/Promtail para reduzir consumo de memória
(viabiliza deploy em VPS de 4GB RAM)
- Provisiona datasources e 4 dashboards Grafana (API Overview, Scraper
Health, Infraestrutura, Business)
- Adiciona requestId middleware no backend para rastreamento de
requisições

### 2. Remoção do vendoring no scraper-go
- Corrige o Dockerfile do scraper-go para usar `go mod download` em vez
de `vendor/`
- Remove o diretório `scraper-go/vendor/` (~147 mil linhas), que não é
mais necessário com o novo fluxo de build

## ⚠️ Nota sobre o tamanho do diff

A maior parte das **412 arquivos alterados** é deleção de arquivos
vendorizados de dependências Go (`scraper-go/vendor/`) — não é código
novo. As mudanças reais de observabilidade estão concentradas em:

- `docker-compose.observability.yml`
- `observability/**`
- `backend/src/metrics/`, `backend/src/middleware/metrics.ts`,
`backend/src/middleware/requestId.ts`
- `backend/src/app.ts` (integração dos middlewares)
- `scraper-go/internal/metrics/`
- `scraper-go/internal/pipeline/pipeline.go` (instrumentação)
- `scraper-go/cmd/server/server.go` (endpoint /metrics)
- `scraper-go/Dockerfile` (build sem vendor)

Esta branch também inclui um merge de `upstream/develop` para resolver
divergência de histórico (a branch foi criada antes da integração de
OAuth GitHub/Google/LinkedIn ter sido mesclada), o que trouxe commits
adicionais que não são de minha autoria.

## Como testar

```bash
docker compose -f docker-compose.infra.yml \
               -f docker-compose.yml \
               -f docker-compose.observability.yml \
               up -d
```

- Backend metrics: `http://localhost:3001/metrics`
- Scraper-go metrics: `http://localhost:8081/metrics`
- Prometheus: `http://localhost:9091`
- Grafana: `http://localhost:3002` (dashboards em "Jobs Scraper Global")

Author: Benevanio

.cspell/custom-dictionary-workspace.txt

@@ -12,9 +12,9 @@ post {
 
 body:json {
   {
-    "email":"hudsonteste@email.com",
-    "password": "teste1234", 
-    "name": "Tavares"
+    "email":"testeregister@email.com",
+    "password": "register1234", 
+    "name": "Register"
   }
 }
 

backend/bruno/Auth/Register.bru

@@ -5,13 +5,13 @@ meta {
 }
 
 get {
-  url: {{base_url}}/jobs/search
+  url: {{base_url}}/jobs/search?keywords=node.js
   body: none
   auth: inherit
 }
 
 params:query {
-  ~keywords: typescript
+  keywords: node.js
 }
 
 headers {

backend/bruno/Jobs/SearchJobs.bru

@@ -1,4 +1,4 @@
 vars {
-  base_url: http://jobsglobalscraper.ddns.net/api
+  base_url: http://localhost:3001/api
   sessionCookie: vagas_session=Fe26.2*1*6b1ee4abcbe0ef8e42e8203e5e9110ead280c442ba3c9e1bf78d0c1f24fb5f5d*7c9X00fVX_sg3MhS9HZe6w*TPGS8-0VdhkyeqTAGGkuJB-4mBhf8ybA4wEB5RrO0S-tC9ykpwRVP8y1b1T8cQFYOPsm7Q-9dccgg92xnaMswi9GoU3Q55N1flcZHo_r7PmLsF8EsadRSraRo5TQOjG2isq6B6UavP_zf_Zr5jkpbA*1780502414077*3727896ffed5c4dd607f5d8460989c13de4218813e5d6263848dc882ae4a9027*Q7YEyyEcQbXbMzCdy4gyJU4RBccUkj6GKw5WW8JhzmE~2
 }

backend/bruno/environments/Local.bru

@@ -43,6 +43,7 @@
     "express": "^5.2.1",
     "iron-session": "^8.0.4",
     "pdfkit": "^0.18.0",
+    "prom-client": "^15.1.3",
     "redis": "^5.12.1",
     "swagger-jsdoc": "^6.3.0",
     "swagger-ui-express": "^5.0.1",

backend/package.json

@@ -1,7 +1,10 @@
 import cors from "cors";
 import express, { NextFunction, Request, Response } from "express";
+import { register } from "./metrics/metrics";
 import { corsOptions } from "./middleware/cors";
 import { errorHandler } from "./middleware/errorHandler";
+import { metricsMiddleware } from "./middleware/metrics";
+import { requestIdMiddleware } from "./middleware/requestId";
 import { requireAuth } from "./middleware/requireAuth";
 import { securityHeaders } from "./middleware/securityHeaders";
 import { withSession } from "./middleware/withSession";
@@ -17,6 +20,8 @@ export function createJobsApiApp() {
   app.disable("x-powered-by");
   app.use(express.json({ limit: "16kb" }));
   app.use(securityHeaders);
+  app.use(requestIdMiddleware);
+  app.use(metricsMiddleware);
 
   // um wrapper para garantir se o CORS chega com erro 403
   app.use((req: Request, res: Response, next: NextFunction) => {
@@ -46,6 +51,11 @@ export function createJobsApiApp() {
    */
   app.get("/api/health", (_req, res) => res.json({ ok: true }));
 
+  app.get("/metrics", async (_req, res) => {
+    res.set("Content-Type", register.contentType);
+    res.end(await register.metrics());
+  });
+
   app.use(errorHandler);
 
   return app;

backend/src/app.ts

@@ -3,6 +3,10 @@ import pino from "pino";
 export const logger = pino({
   level: process.env.LOG_LEVEL ?? "info",
   timestamp: pino.stdTimeFunctions.isoTime,
+  base: {
+    service: "backend",
+    environment: process.env.NODE_ENV ?? "development",
+  },
   formatters: {
     level(label) {
       return { level: label };
@@ -16,3 +20,11 @@ export const logWarn = (msg: string, ctx?: object) =>
   logger.warn(ctx ?? {}, msg);
 export const logError = (msg: string, ctx?: object) =>
   logger.error(ctx ?? {}, msg);
+
+/**
+ * Cria um child logger com requestId/userId já anexados,
+ * para usar dentro de rotas/controllers.
+ */
+export function getRequestLogger(ctx: { requestId?: string; userId?: string }) {
+  return logger.child(ctx);
+}

backend/src/logger.ts

@@ -0,0 +1,39 @@
+import client from "prom-client";
+
+export const register = new client.Registry();
+client.collectDefaultMetrics({ register });
+
+export const httpRequestDuration = new client.Histogram({
+  name: "http_request_duration_seconds",
+  help: "Duração das requisições HTTP",
+  labelNames: ["method", "route", "status_code"],
+  buckets: [0.01, 0.05, 0.1, 0.3, 0.5, 1, 2, 5],
+  registers: [register],
+});
+
+export const httpRequestsTotal = new client.Counter({
+  name: "http_requests_total",
+  help: "Total de requisições HTTP",
+  labelNames: ["method", "route", "status_code"],
+  registers: [register],
+});
+
+export const activeSessions = new client.Gauge({
+  name: "active_sessions",
+  help: "Sessões ativas (aproximação por requisições autenticadas recentes)",
+  registers: [register],
+});
+
+export const jobSearchesTotal = new client.Counter({
+  name: "job_searches_total",
+  help: "Total de buscas de vagas realizadas",
+  labelNames: ["has_keywords"],
+  registers: [register],
+});
+
+export const cacheOperationsTotal = new client.Counter({
+  name: "cache_operations_total",
+  help: "Operações de cache no Valkey",
+  labelNames: ["operation", "result"],
+  registers: [register],
+});

backend/src/metrics/metrics.ts

@@ -0,0 +1,28 @@
+import type { NextFunction, Request, Response } from "express";
+import { httpRequestDuration, httpRequestsTotal } from "../metrics/metrics";
+
+export function metricsMiddleware(
+  req: Request,
+  res: Response,
+  next: NextFunction,
+) {
+  const end = httpRequestDuration.startTimer();
+
+  res.on("finish", () => {
+    // route já resolvido pelo Express (com :params), com fallback pro path cru
+    const route = req.route?.path
+      ? `${req.baseUrl}${req.route.path}`
+      : req.path;
+
+    const labels = {
+      method: req.method,
+      route,
+      status_code: String(res.statusCode),
+    };
+
+    end(labels);
+    httpRequestsTotal.inc(labels);
+  });
+
+  next();
+}

backend/src/middleware/metrics.ts

@@ -0,0 +1,18 @@
+import { randomUUID } from "crypto";
+import type { NextFunction, Request, Response } from "express";
+
+declare module "express-serve-static-core" {
+  interface Request {
+    requestId: string;
+  }
+}
+
+export function requestIdMiddleware(
+  req: Request,
+  res: Response,
+  next: NextFunction,
+) {
+  req.requestId = (req.headers["x-request-id"] as string) || randomUUID();
+  res.setHeader("x-request-id", req.requestId);
+  next();
+}