docker-compose.yml

#================================================================================================
# TelemetryFlow Core - Docker Compose Configuration
# Full Stack: PostgreSQL + ClickHouse + Redis + NATS + Backend + Frontend
#
# PROFILES:
# - core: Essential services (postgres, clickhouse, redis, nats, backend, frontend)
# - monitoring: Observability stack (jaeger)
# - tools: Management tools (portainer)
# - all: All services combined
#
# USAGE:
# docker-compose --profile core up -d                         # Core services only
# docker-compose --profile core --profile monitoring up -d    # Core + monitoring
# docker-compose --profile all up -d                          # All services
#================================================================================================

networks:
  telemetryflow_core_net:
    name: telemetryflow_core_net
    driver: bridge
    ipam:
      config:
        - subnet: 172.154.0.0/16

services:

  #----------------------------------------------------------------------------------------------
  # POSTGRESQL
  #----------------------------------------------------------------------------------------------
  postgres:
    profiles: ["core", "all"]
    platform: linux/amd64
    image: postgres:${POSTGRES_VERSION:-16-alpine}
    container_name: ${CONTAINER_POSTGRES:-telemetryflow_core_postgres}
    restart: unless-stopped
    ports:
      - "${PORT_POSTGRES:-5432}:5432"
    environment:
      - TZ=${TZ:-UTC}
      - POSTGRES_USER=${POSTGRES_USERNAME:-postgres}
      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-}
      - POSTGRES_DB=${POSTGRES_DB:-telemetryflow_db}
      - PGDATA=/var/lib/postgresql/data/pgdata
    volumes:
      - ${VOLUMES_BASE_PATH:-/opt/data/docker/telemetryflow-core}/postgresql:/var/lib/postgresql/data
      - ./config/postgresql/postgresql.conf:/etc/postgresql/postgresql.conf:ro
    networks:
      telemetryflow_core_net:
        ipv4_address: ${CONTAINER_IP_POSTGRES:-172.154.154.20}
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USERNAME:-postgres} -d ${POSTGRES_DB:-telemetryflow_db}"]
      interval: 10s
      timeout: 5s
      retries: 5
      start_period: 10s

  #----------------------------------------------------------------------------------------------
  # CLICKHOUSE - Time-Series & Analytics Storage (Metrics, Logs, Traces, Audit)
  #----------------------------------------------------------------------------------------------
  clickhouse:
    profiles: ["core", "all"]
    platform: linux/amd64
    image: clickhouse/clickhouse-server:${CLICKHOUSE_VERSION:-latest}
    container_name: ${CONTAINER_CLICKHOUSE:-telemetryflow_core_clickhouse}
    restart: unless-stopped
    ports:
      - "${PORT_CLICKHOUSE_HTTP:-8123}:8123"    # HTTP
      - "${PORT_CLICKHOUSE_NATIVE:-9000}:9000"  # Native
      - "${PORT_CLICKHOUSE_METRICS:-9363}:9363" # Prometheus metrics
    environment:
      - TZ=${TZ:-UTC}
      - CLICKHOUSE_DB=${CLICKHOUSE_DB:-telemetryflow_db}
      - CLICKHOUSE_USER=${CLICKHOUSE_USER:-default}
      - CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD:-}
    volumes:
      - ${DATA_CLICKHOUSE:-${VOLUMES_BASE_PATH:-/opt/data/docker/telemetryflow-core}/clickhouse}/data:/var/lib/clickhouse
      - ${DATA_CLICKHOUSE:-${VOLUMES_BASE_PATH:-/opt/data/docker/telemetryflow-core}/clickhouse}/logs:/var/log/clickhouse-server
    # - ./config/clickhouse/config.d:/etc/clickhouse-server/config.d
    networks:
      telemetryflow_core_net:
        ipv4_address: ${CONTAINER_IP_CLICKHOUSE:-172.151.151.40}
    healthcheck:
      test: ["CMD", "wget", "--spider", "-q", "http://localhost:8123/ping"]
      interval: 10s
      timeout: 5s
      retries: 5
      start_period: 10s

  #----------------------------------------------------------------------------------------------
  # REDIS - Cache, Sessions, BullMQ Queues
  #----------------------------------------------------------------------------------------------
  redis:
    profiles: ["core", "all"]
    platform: linux/amd64
    image: redis:7-alpine
    container_name: ${CONTAINER_REDIS:-telemetryflow_core_redis}
    restart: unless-stopped
    ports:
      - "${PORT_REDIS:-6379}:6379"
    command: >
      redis-server
      --maxmemory ${REDIS_MAX_MEMORY:-512mb}
      --maxmemory-policy noeviction
      --save 60 1000
      --appendonly yes
    volumes:
      - ${VOLUMES_BASE_PATH:-/opt/data/docker/telemetryflow-core}/redis:/data
    networks:
      telemetryflow_core_net:
        ipv4_address: ${CONTAINER_IP_REDIS:-172.154.154.30}
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 10s
      timeout: 5s
      retries: 5

  #----------------------------------------------------------------------------------------------
  # NATS - Cross-Module Domain Event Messaging
  #   JetStream enabled for durable message delivery
  #----------------------------------------------------------------------------------------------
  nats:
    profiles: ["core", "all"]
    platform: linux/amd64
    image: nats:${NATS_VERSION:-2-alpine}
    container_name: ${CONTAINER_NATS:-telemetryflow_core_nats}
    restart: unless-stopped
    ports:
      - "${PORT_NATS_CLIENT:-4222}:4222"   # Client connections
      - "${PORT_NATS_MONITOR:-8222}:8222"  # HTTP monitoring
    command: ["--js", "--sd", "/data", "-m", "8222"]
    volumes:
      - ${DATA_NATS:-${VOLUMES_BASE_PATH:-/opt/data/docker/telemetryflow-core}/nats}:/data
    networks:
      telemetryflow_core_net:
        ipv4_address: ${CONTAINER_IP_NATS:-172.154.154.55}
    healthcheck:
      test: ["CMD", "wget", "--spider", "-q", "http://localhost:8222/healthz"]
      interval: 10s
      timeout: 5s
      retries: 5
      start_period: 5s

  #----------------------------------------------------------------------------------------------
  # BACKEND - NestJS API
  #----------------------------------------------------------------------------------------------
  backend:
    profiles: ["core"]
    platform: linux/amd64
    build:
      context: .
      dockerfile: Dockerfile.backend
    container_name: ${CONTAINER_BACKEND:-telemetryflow_core_backend}
    restart: unless-stopped
    ports:
      - "${PORT_BACKEND:-3000}:3000"
    environment:
      - TZ=${TZ:-UTC}
      - NODE_ENV=${NODE_ENV:-production}
      - PORT=3000
      - POSTGRES_HOST=${POSTGRES_HOST:-postgres}
      - POSTGRES_PORT=5432
      - POSTGRES_USERNAME=${POSTGRES_USERNAME:-postgres}
      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-}
      - POSTGRES_DB=${POSTGRES_DB:-telemetryflow_db}
      - CLICKHOUSE_HOST=${CLICKHOUSE_HOST:-clickhouse}
      - CLICKHOUSE_PORT=8123
      - CLICKHOUSE_DB=${CLICKHOUSE_DB:-telemetryflow_db}
      - CLICKHOUSE_USER=${CLICKHOUSE_USER:-default}
      - CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD:-}
      - REDIS_HOST=${REDIS_HOST:-redis}
      - REDIS_PORT=6379
      - REDIS_PASSWORD=${REDIS_PASSWORD:-}
      - REDIS_CACHE_DB=0
      - REDIS_QUEUE_DB=1
      - NATS_URL=nats://nats:4222
      - NATS_CLUSTER_ID=${NATS_CLUSTER_ID:-telemetryflow-cluster}
      - JWT_SECRET=${JWT_SECRET}
      - JWT_REFRESH_SECRET=${JWT_REFRESH_SECRET}
      - SESSION_SECRET=${SESSION_SECRET}
      - ENCRYPTION_KEY=${ENCRYPTION_KEY}
      - MFA_ENCRYPTION_KEY=${MFA_ENCRYPTION_KEY}
      - LLM_ENCRYPTION_KEY=${LLM_ENCRYPTION_KEY}
      - CORS_ORIGIN=${CORS_ORIGIN:-*}
      - LOGGER_TYPE=${LOGGER_TYPE:-winston}
      - OTEL_ENABLED=${OTEL_ENABLED:-false}
      - SMTP_ENABLED=${SMTP_ENABLED:-false}
    depends_on:
      postgres:
        condition: service_healthy
      clickhouse:
        condition: service_healthy
      redis:
        condition: service_healthy
      nats:
        condition: service_healthy
    networks:
      telemetryflow_core_net:
        ipv4_address: ${CONTAINER_IP_BACKEND:-172.154.154.10}

  #----------------------------------------------------------------------------------------------
  # FRONTEND - Vue 3 SPA via Nginx
  #----------------------------------------------------------------------------------------------
  frontend:
    profiles: ["core"]
    platform: linux/amd64
    build:
      context: .
      dockerfile: Dockerfile.frontend
    container_name: ${CONTAINER_FRONTEND:-telemetryflow_core_frontend}
    restart: unless-stopped
    ports:
      - "${PORT_FRONTEND:-8080}:80"
    depends_on:
      - backend
    networks:
      telemetryflow_core_net:
        ipv4_address: ${CONTAINER_IP_FRONTEND:-172.154.154.80}

  #==============================================================================================
  # MONITORING SERVICES
  #==============================================================================================

  #----------------------------------------------------------------------------------------------
  # OTEL COLLECTOR - OpenTelemetry Collector
  #----------------------------------------------------------------------------------------------
  # Supports dual ingestion: v1 (OTEL standard) and v2 (TelemetryFlow enhanced)
  #
  # OTLP HTTP Endpoints:
  #   TelemetryFlow Platform (Recommended - TFO Standalone):
  #     POST http://localhost:4318/v2/traces
  #     POST http://localhost:4318/v2/metrics
  #     POST http://localhost:4318/v2/logs
  #
  #   OTEL Community (Backwards Compatible - OCB/Standard):
  #     POST http://localhost:4318/v1/traces
  #     POST http://localhost:4318/v1/metrics
  #     POST http://localhost:4318/v1/logs
  #
  #   gRPC (Both versions via same port): localhost:4317
  #----------------------------------------------------------------------------------------------
  tfo-collector:
    profiles: ["monitoring", "all"]
    platform: linux/amd64
    # =============================================================================
    # Default: OTEL Collector Community (public image, works everywhere)
    # image: otel/opentelemetry-collector-contrib:${OTEL_VERSION:-0.116.0}
    # command: ["--config=/etc/otelcol-contrib/config.yaml"]
    # =============================================================================
    # TelemetryFlow Collector (TFO-Collector) - Custom TFO format (v1.1.2+)
    image: telemetryflow/telemetryflow-collector:${TFO_COLLECTOR_VERSION:-1.2.1}
    command: ["--config=/etc/tfo-collector/tfo-collector.yaml"]
    # =============================================================================
    container_name: ${CONTAINER_OTEL:-telemetryflow_core_tfo_collector}
    restart: unless-stopped
    environment:
      # Backend endpoint for tfo exporter (backend runs on host, not Docker)
      - TELEMETRYFLOW_ENDPOINT=http://host.docker.internal:3000
      # API keys for tfoauth extension
      - TELEMETRYFLOW_API_KEY_ID=${TELEMETRYFLOW_API_KEY_ID}
      - TELEMETRYFLOW_API_KEY_SECRET=${TELEMETRYFLOW_API_KEY_SECRET}
      # Collector identity
      - TELEMETRYFLOW_COLLECTOR_ID=${CONTAINER_OTEL:-telemetryflow_core_tfo_collector}
      - TELEMETRYFLOW_ENVIRONMENT=development
    volumes:
      # TelemetryFlow Collector config (Custom TFO format — used by tfo-collector image)
      - ./config/tfo-collector/tfo-collector.yaml:/etc/tfo-collector/tfo-collector.yaml:ro
      # =============================================================================
      # OTEL Community Collector config (uncomment if using otel/opentelemetry-collector-contrib image)
      # - ./config/tfo-collector/otel-collector.yaml:/etc/otelcol-contrib/config.yaml:ro
    ports:
      - "${PORT_OTEL_GRPC:-4317}:4317"     # OTLP gRPC (v1 & v2)
      - "${PORT_OTEL_HTTP:-4318}:4318"     # OTLP HTTP (v1 & v2)
      - "${PORT_OTEL_METRICS:-8889}:8889"  # Prometheus metrics
      - "${PORT_OTEL_HEALTH:-13133}:13133" # Health check
      - "${PORT_OTEL_ZPAGES:-55679}:55679" # zPages
      - "${PORT_OTEL_PPROF:-1777}:1777"    # pprof
    extra_hosts:
      - "host.docker.internal:host-gateway"
    healthcheck:
      test: ["CMD", "wget", "--spider", "-q", "http://localhost:13133"]
      interval: 10s
      timeout: 5s
      retries: 5
      start_period: 10s
    networks:
      telemetryflow_core_net:
        ipv4_address: ${CONTAINER_IP_OTEL:-172.151.151.30}

  #----------------------------------------------------------------------------------------------
  # TFO-AGENT - TelemetryFlow Host Metrics Agent
  #   Collects host-level metrics (CPU, memory, disk, network) and sends heartbeats
  #   to the backend API. Replaces Prometheus for infrastructure monitoring.
  #----------------------------------------------------------------------------------------------
  tfo-agent:
    profiles: ["monitoring", "all"]
    platform: linux/amd64
    image: telemetryflow/telemetryflow-agent:${TFO_AGENT_VERSION:-1.2.0}
    container_name: ${CONTAINER_TFO_AGENT:-telemetryflow_core_tfo_agent}
    hostname: ${TFO_AGENT_HOSTNAME:-tfo-agent}
    restart: unless-stopped
    ports:
      - "${PORT_TFO_AGENT:-9191}:9191"   # Health & metrics endpoint
    environment:
      - TZ=${TZ:-UTC}
      # Platform backend for heartbeat/registration (host.docker.internal → host port 3000)
      - TELEMETRYFLOW_ENDPOINT=http://host.docker.internal:3000/api/v2
      # OTLP data goes through the collector (collector forwards to backend via tfo exporter)
      - TELEMETRYFLOW_OTLP_ENDPOINT=http://tfo-collector:4318
      - TELEMETRYFLOW_API_KEY_ID=${TELEMETRYFLOW_API_KEY_ID}
      - TELEMETRYFLOW_API_KEY_SECRET=${TELEMETRYFLOW_API_KEY_SECRET}
      - TELEMETRYFLOW_AGENT_ID=${CONTAINER_TFO_AGENT:-telemetryflow_core_tfo_agent}
      - TELEMETRYFLOW_AGENT_NAME=TFO Docker Agent
      - TELEMETRYFLOW_ENVIRONMENT=development
      # Intervals
      - TFO_AGENT_HEARTBEAT_INTERVAL=${TFO_AGENT_HEARTBEAT_INTERVAL:-60s}
      - TFO_AGENT_METRICS_INTERVAL=${TFO_AGENT_METRICS_INTERVAL:-30s}
    volumes:
      - ./config/tfo-agent/tfo-agent.yaml:/etc/tfo-agent/tfo-agent.yaml:ro
      - ${DATA_TFO_AGENT:-${VOLUMES_BASE_PATH:-/opt/data/docker/telemetryflow-platform}/tfo-agent}:/data
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /var/log:/var/log:ro              # Host logs for Fluent Bit / native log collector
    extra_hosts:
      - "host.docker.internal:host-gateway"
    depends_on:
      tfo-collector:
        condition: service_healthy
    networks:
      telemetryflow_core_net:
        ipv4_address: ${CONTAINER_IP_TFO_AGENT:-172.151.151.35}
    healthcheck:
      test: ["CMD-SHELL", "wget --spider -q http://localhost:13133/ || exit 1"]
      interval: 30s
      timeout: 5s
      retries: 3
      start_period: 15s

  #----------------------------------------------------------------------------------------------
  # PORTAINER - Docker Management (optional)
  #----------------------------------------------------------------------------------------------
  portainer:
    profiles: ["tools", "all"]
    platform: linux/amd64
    image: portainer/portainer-ce:${PORTAINER_VERSION:-latest}
    container_name: ${CONTAINER_PORTAINER:-telemetryflow_core_portainer}
    restart: unless-stopped
    ports:
      - "${PORT_PORTAINER:-9100}:9000"
      - "${PORT_PORTAINER_HTTPS:-9443}:9443"
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - ${VOLUMES_BASE_PATH:-/opt/data/docker/telemetryflow-core}/portainer:/data
    networks:
      telemetryflow_core_net:
        ipv4_address: ${CONTAINER_IP_PORTAINER:-172.154.154.5}