chore: 슬랙 웹 훅 URL 값을 도플러로 분리, alertmanager 리포지토리 등록 (#539)

* fix: 프로메테우스 v3 오탈자 수정

* chore: 슬랙 웹 훅 URL 값을 도플러로 분리. alertmanager 리포지토리 등록.

Author: weilim0513-tech

.github/workflows/deploy.yml

@@ -135,6 +135,10 @@ jobs:
               "GITHUB_TOKEN=$(aws ssm get-parameter --name /tt/github-token --with-decryption --query Parameter.Value --output text --region ap-northeast-2)",
               "echo \"$GITHUB_TOKEN\" | docker login ghcr.io -u ${{ github.actor }} --password-stdin 2>/dev/null",
 
+              "set -a && source doppler.env && set +a",
+              "envsubst < monitoring/alertmanager/alertmanager.yml > /tmp/alertmanager-resolved.yml",
+              "cp /tmp/alertmanager-resolved.yml monitoring/alertmanager/alertmanager.yml",
+
               "doppler run --project \"$DOPPLER_PROJECT\" --config \"$DOPPLER_CONFIG\" -- docker compose pull",
               "doppler run --project \"$DOPPLER_PROJECT\" --config \"$DOPPLER_CONFIG\" -- docker compose up -d --force-recreate",
 

.gitignore

@@ -60,6 +60,7 @@ src/main/generated/
 !docker/monitoring/prometheus/prometheus.yml
 !docker/monitoring/prometheus/rules/*.yml
 !docker/monitoring/grafana/provisioning/**/*.yml
+!docker/monitoring/alertmanager/alertmanager.yml
 
 ### custom ###
 src/main/resources/application-secret.yml

docker/docker-compose.yml

@@ -396,7 +396,6 @@ services:
     ports:
       - "9093:9093"
     environment:
-      - SLACK_WEBHOOK_URL=${SLACK_WEBHOOK_URL}
     volumes:
       - ./monitoring/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
       - alertmanager_data:/alertmanager
@@ -462,4 +461,4 @@ volumes:
   redis-node-6-data:
   prometheus_data:
   grafana_data:
-  alertmanager_data:
+  alertmanager_data:

docker/monitoring/alertmanager/alertmanager.yml

@@ -0,0 +1,121 @@
+# 각 도메인은 자신의 SLACK_WEBHOOK_XXX 환경변수 사용
+
+global:
+  resolve_timeout: 5m
+
+# 라우팅 설정
+route:
+  receiver: 'slack-auth-default'
+  group_by: [ 'alertname', 'severity', 'domain' ]
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 4h
+
+  routes:
+    # ==================== CRITICAL (즉시 알림) ====================
+    # 계정 잠금 폭증 - 공격 가능성
+    - match:
+        alertname: HighAccountLockRate
+      receiver: 'slack-security-critical'
+      group_wait: 10s
+      repeat_interval: 30m
+
+    # 보안 위협 탐지
+    - match:
+        alertname: SecurityThreatDetected
+      receiver: 'slack-security-critical'
+      group_wait: 10s
+      repeat_interval: 30m
+
+    # 로그인 실패 급증 - Brute Force 가능성
+    - match:
+        alertname: LoginFailureSpike
+      receiver: 'slack-security-critical'
+      group_wait: 10s
+      repeat_interval: 30m
+
+    # ==================== WARNING ====================
+    # 로그인 성공률 저하
+    - match:
+        alertname: LowLoginSuccessRate
+      receiver: 'slack-auth-warning'
+      repeat_interval: 2h
+
+    # 이메일 발송 실패율 증가
+    - match:
+        alertname: HighEmailFailureRate
+      receiver: 'slack-email-warning'
+      repeat_interval: 2h
+
+    # 이메일 인증 실패 급증
+    - match:
+        alertname: HighVerificationFailureRate
+      receiver: 'slack-email-warning'
+      repeat_interval: 2h
+
+    # Rate Limit 다수 트리거
+    - match:
+        alertname: RateLimitTriggered
+      receiver: 'slack-auth-warning'
+      repeat_interval: 1h
+
+    # ==================== INFO ====================
+    # 회원가입/탈퇴 통계
+    - match:
+        domain: member
+        severity: info
+      receiver: 'slack-member-info'
+      repeat_interval: 24h
+
+# ==================== Auth 도메인 수신자 ====================
+receivers:
+  # 기본 (라우팅 안 된 알림)
+  - name: 'slack-auth-default'
+    slack_configs:
+      - api_url: '${SLACK_WEBHOOK_AUTH}'
+        channel: '#tt-auth-alerts'
+        send_resolved: true
+
+  # 보안 Critical - 즉시 대응 필요
+  - name: 'slack-security-critical'
+    slack_configs:
+      - api_url: '${SLACK_WEBHOOK_AUTH}'
+        channel: '#tt-auth-alerts'
+        send_resolved: true
+
+  # Auth Warning - 주의 필요
+  - name: 'slack-auth-warning'
+    slack_configs:
+      - api_url: '${SLACK_WEBHOOK_AUTH}'
+        channel: '#tt-auth-alerts'
+        send_resolved: true
+
+  # Email Warning
+  - name: 'slack-email-warning'
+    slack_configs:
+      - api_url: '${SLACK_WEBHOOK_AUTH}'
+        channel: '#tt-auth-alerts'
+        send_resolved: true
+
+  # Member 정보성 알림 (일간 리포트 등)
+  - name: 'slack-member-info'
+    slack_configs:
+      - api_url: '${SLACK_WEBHOOK_AUTH}'
+        channel: '#tt-auth-alerts'
+        send_resolved: false
+
+# 알림 억제 규칙
+inhibit_rules:
+  # Critical 있으면 같은 alertname의 Warning 억제
+  - source_match:
+      severity: 'critical'
+    target_match:
+      severity: 'warning'
+    equal: [ 'alertname' ]
+
+  # 전체 서비스 다운이면 개별 알림 억제
+  - source_match:
+      alertname: 'ServiceDown'
+    target_match_re:
+      alertname: '.+'
+    equal: [ 'instance' ]